35#include <sys/eventhandler.h>
36#include <sys/kernel.h>
38#include <sys/malloc.h>
39#include <sys/module.h>
45#include <sys/sysctl.h>
48#include <sys/timetc.h>
49#include <sys/taskqueue.h>
51#include "cpufreq_if.h"
62#define CF_MAX_LEVELS 256
75 struct cf_level_lst all_levels;
80 struct sysctl_ctx_list sysctl_ctx;
81 struct task startup_task;
82 struct cf_level *levels_buf;
86 struct cf_setting
sets[MAX_SETTINGS];
93#define CF_MTX_INIT(x) sx_init((x), "cpufreq lock")
94#define CF_MTX_LOCK(x) sx_xlock((x))
95#define CF_MTX_UNLOCK(x) sx_xunlock((x))
96#define CF_MTX_ASSERT(x) sx_assert((x), SX_XLOCKED)
98#define CF_DEBUG(msg...) do { \
100 printf("cpufreq: " msg); \
116 struct cf_level *dup,
struct cf_setting *
set);
140 "cpufreq debugging");
142 "Don't provide levels below this frequency.");
144 "Print verbose debugging messages");
164 TAILQ_INIT(&sc->all_levels);
166 sc->
curr_level.total_set.freq = CPUFREQ_VAL_UNKNOWN;
167 SLIST_INIT(&sc->saved_freq);
169 sc->max_mhz = cpu_get_nominal_mhz(dev);
171 if (sc->max_mhz <= 0) {
172 CF_DEBUG(
"Unable to obtain nominal frequency.\n");
173 pc = cpu_get_pcpu(dev);
174 if (cpu_est_clockrate(pc->pc_cpuid, &rate) == 0)
175 sc->max_mhz = rate / 1000000;
177 sc->max_mhz = CPUFREQ_VAL_UNKNOWN;
180 CF_DEBUG(
"initializing one-time data for %s\n",
184 SYSCTL_ADD_PROC(&sc->sysctl_ctx,
186 OID_AUTO,
"freq", CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_NEEDGIANT,
188 SYSCTL_ADD_PROC(&sc->sysctl_ctx,
190 OID_AUTO,
"freq_levels",
191 CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_NEEDGIANT, sc, 0,
222 while ((saved_freq = SLIST_FIRST(&sc->saved_freq)) != NULL) {
223 SLIST_REMOVE_HEAD(&sc->saved_freq, link);
224 free(saved_freq, M_TEMP);
227 free(sc->levels_buf, M_DEVBUF);
236 const struct cf_setting *
set;
248 EVENTHANDLER_INVOKE(cpufreq_pre_change,
level, &error);
250 EVENTHANDLER_INVOKE(cpufreq_post_change,
level, error);
257#ifdef EARLY_AP_STARTUP
268 device_printf(dev,
"rejecting change, SMP not started yet\n");
279 if (priority < sc->curr_priority) {
291 saved_freq = SLIST_FIRST(&sc->saved_freq);
292 if (saved_freq == NULL) {
293 CF_DEBUG(
"NULL level, no saved level\n");
299 CF_DEBUG(
"restoring saved level, freq %d prio %d\n",
305 CF_DEBUG(
"rejecting freq %d, less than %d limit\n",
313 CF_DEBUG(
"skipping freq %d, same as current level %d\n",
327 pc = cpu_get_pcpu(
set->dev);
334 thread_lock(curthread);
335 pri = curthread->td_priority;
338 thread_unlock(curthread);
339 CF_DEBUG(
"setting abs freq %d on %s (cpu %d)\n",
set->freq,
341 error = CPUFREQ_DRV_SET(
set->dev,
set);
342 thread_lock(curthread);
345 thread_unlock(curthread);
352 for (i = 0; i <
level->rel_count; i++) {
360 pc = cpu_get_pcpu(
set->dev);
361 thread_lock(curthread);
362 pri = curthread->td_priority;
365 thread_unlock(curthread);
366 CF_DEBUG(
"setting rel freq %d on %s (cpu %d)\n",
set->freq,
368 error = CPUFREQ_DRV_SET(
set->dev,
set);
369 thread_lock(curthread);
372 thread_unlock(curthread);
384 if (sc->
curr_level.total_set.freq != CPUFREQ_VAL_UNKNOWN &&
386 CF_DEBUG(
"saving level, freq %d prio %d\n",
388 curr_freq =
malloc(
sizeof(*curr_freq), M_TEMP, M_NOWAIT);
389 if (curr_freq == NULL) {
395 SLIST_INSERT_HEAD(&sc->saved_freq, curr_freq, link);
401 if (saved_freq != NULL) {
402 CF_DEBUG(
"resetting saved level\n");
403 sc->
curr_level.total_set.freq = CPUFREQ_VAL_UNKNOWN;
404 SLIST_REMOVE_HEAD(&sc->saved_freq, link);
405 free(saved_freq, M_TEMP);
415 EVENTHANDLER_INVOKE(cpufreq_post_change,
level, error);
425 struct cf_setting
set;
427 if (CPUFREQ_DRV_GET(dev, &
set) != 0)
441 for (i = 0; i <
count; i++)
442 if (freq ==
levels[i].total_set.freq)
459 struct cf_setting *curr_set;
474 error = CPUFREQ_DRV_TYPE(sc->cf_drv_dev, &
type);
475 if (error == 0 && (
type & CPUFREQ_FLAG_UNCACHED)) {
476 struct cf_setting
set;
483 if (CPUFREQ_DRV_GET(sc->cf_drv_dev, &
set) == 0) {
485 CF_DEBUG(
"get returning immediate freq %d\n",
489 }
else if (curr_set->freq != CPUFREQ_VAL_UNKNOWN) {
490 CF_DEBUG(
"get returning known freq %d\n", curr_set->freq);
508 printf(
"cpufreq: need to increase CF_MAX_LEVELS\n");
526 CF_DEBUG(
"Couldn't find supported level for %s\n",
529 if (curr_set->freq != CPUFREQ_VAL_UNKNOWN) {
530 CF_DEBUG(
"get matched freq %d from drivers\n", curr_set->freq);
538 pc = cpu_get_pcpu(dev);
543 cpu_est_clockrate(pc->pc_cpuid, &rate);
546 for (i = 0; i <
count; i++) {
547 diff = abs(
levels[i].total_set.freq - rate);
553 CF_DEBUG(
"get estimated freq %d\n", curr_set->freq);
573 struct cf_setting *
sets;
576 int type, set_count, error;
579 dev = sc->cf_drv_dev;
589 error = CPUFREQ_DRV_TYPE(dev, &
type);
590 if (error != 0 || (
type & CPUFREQ_FLAG_INFO_ONLY)) {
592 CF_DEBUG(
"skipping info-only driver %s\n",
602 set_count = MAX_SETTINGS;
603 error = CPUFREQ_DRV_SETTINGS(dev,
sets, &set_count);
604 if (error != 0 || set_count == 0)
608 switch (
type & CPUFREQ_TYPE_MASK) {
609 case CPUFREQ_TYPE_ABSOLUTE:
612 case CPUFREQ_TYPE_RELATIVE:
613 CF_DEBUG(
"adding %d relative settings\n", set_count);
614 set_arr =
malloc(
sizeof(*set_arr), M_TEMP, M_NOWAIT);
615 if (set_arr == NULL) {
620 set_arr->
count = set_count;
621 TAILQ_INSERT_TAIL(rel_sets, set_arr, link);
636 struct cf_setting_lst rel_sets;
638 struct cf_level *lev;
646 TAILQ_INIT(&rel_sets);
658 if (TAILQ_EMPTY(&sc->all_levels)) {
659 struct cf_setting
set;
661 CF_DEBUG(
"No absolute levels returned by driver\n");
663 if (sc->max_mhz == CPUFREQ_VAL_UNKNOWN) {
664 sc->max_mhz = cpu_get_nominal_mhz(dev);
670 if (sc->max_mhz <= 0) {
671 pc = cpu_get_pcpu(dev);
672 cpu_est_clockrate(pc->pc_cpuid, &rate);
673 sc->max_mhz = rate / 1000000;
676 memset(&
set, CPUFREQ_VAL_UNKNOWN,
sizeof(
set));
677 set.freq = sc->max_mhz;
685 TAILQ_FOREACH(set_arr, &rel_sets, link)
689 if (sc->all_count > *
count) {
690 *
count = sc->all_count;
697 TAILQ_FOREACH(lev, &sc->all_levels, link) {
707 *
count = sc->all_count;
712 while ((lev = TAILQ_FIRST(&sc->all_levels)) != NULL) {
713 TAILQ_REMOVE(&sc->all_levels, lev, link);
719 while ((set_arr = TAILQ_FIRST(&rel_sets)) != NULL) {
720 TAILQ_REMOVE(&rel_sets, set_arr, link);
721 free(set_arr, M_TEMP);
734 struct cf_level_lst *list;
735 struct cf_level *
level, *search;
740 list = &sc->all_levels;
741 for (i = 0; i <
count; i++) {
747 level->total_set.dev = NULL;
751 if (TAILQ_EMPTY(list)) {
752 CF_DEBUG(
"adding abs setting %d at head\n",
754 TAILQ_INSERT_HEAD(list,
level, link);
758 TAILQ_FOREACH_REVERSE(search, list, cf_level_lst, link)
759 if (
sets[i].freq <= search->total_set.freq) {
760 CF_DEBUG(
"adding abs setting %d after %d\n",
761 sets[i].freq, search->total_set.freq);
762 TAILQ_INSERT_AFTER(list, search,
level, link);
768 TAILQ_FOREACH(search, list, link)
769 if (
sets[i].freq >= search->total_set.freq) {
770 CF_DEBUG(
"adding abs setting %d before %d\n",
771 sets[i].freq, search->total_set.freq);
772 TAILQ_INSERT_BEFORE(search,
level, link);
787 struct cf_level *fill, *search;
788 struct cf_setting *
set;
801 TAILQ_FOREACH_REVERSE(search, &sc->all_levels, cf_level_lst, link) {
803 for (i = 0; i < set_arr->
count; i++) {
811 if (
set->freq < 10000) {
826 KASSERT(fill->rel_count < MAX_SETTINGS,
827 (
"cpufreq: too many relative drivers (%d)",
829 fill->rel_set[fill->rel_count] = *
set;
832 "expand set added rel setting %d%% to %d level\n",
833 set->freq / 100, fill->total_set.freq);
840static struct cf_level *
842 struct cf_setting *
set)
844 struct cf_level_lst *list;
845 struct cf_level *fill, *itr;
846 struct cf_setting *fill_set, *itr_set;
856 fill =
malloc(
sizeof(*fill), M_TEMP, M_NOWAIT);
860 fill_set = &fill->total_set;
862 ((uint64_t)fill_set->freq *
set->freq) / 10000;
863 if (fill_set->power != CPUFREQ_VAL_UNKNOWN) {
864 fill_set->power = ((uint64_t)fill_set->power *
set->freq)
867 if (
set->lat != CPUFREQ_VAL_UNKNOWN) {
868 if (fill_set->lat != CPUFREQ_VAL_UNKNOWN)
869 fill_set->lat +=
set->lat;
871 fill_set->lat =
set->lat;
873 CF_DEBUG(
"dup set considering derived setting %d\n", fill_set->freq);
881 for (i = fill->rel_count; i != 0; i--) {
882 if (fill->rel_set[i - 1].dev !=
set->dev)
884 CF_DEBUG(
"removed last relative driver: %s\n",
899 list = &sc->all_levels;
900 KASSERT(!TAILQ_EMPTY(list), (
"all levels list empty in dup set"));
901 TAILQ_FOREACH_REVERSE(itr, list, cf_level_lst, link) {
902 itr_set = &itr->total_set;
903 if (CPUFREQ_CMP(fill_set->freq, itr_set->freq)) {
904 CF_DEBUG(
"dup set rejecting %d (dupe)\n",
908 }
else if (fill_set->freq < itr_set->freq) {
909 if (fill->abs_set.freq <= itr->abs_set.freq) {
911 "dup done, inserting new level %d after %d\n",
912 fill_set->freq, itr_set->freq);
913 TAILQ_INSERT_AFTER(list, itr, fill, link);
916 CF_DEBUG(
"dup set rejecting %d (abs too big)\n",
926 CF_DEBUG(
"dup set freeing new level %d (not optimal)\n",
940 int best,
count, diff, bdiff, devcount, error, freq, i, n;
947 error = CPUFREQ_GET(sc->dev, &
levels[0]);
950 freq =
levels[0].total_set.freq;
952 if (error != 0 || req->newptr == NULL)
963 for (n = 0; n < devcount; n++) {
969 "cpufreq: need to increase CF_MAX_LEVELS\n");
974 for (i = 0; i <
count; i++) {
975 diff = abs(
levels[i].total_set.freq - freq);
981 error = CPUFREQ_SET(devs[n], &
levels[best], CPUFREQ_PRIO_USER);
995 struct cf_setting *
set;
1000 sbuf_new(&sb, NULL, 128, SBUF_AUTOEXTEND);
1012 printf(
"cpufreq: need to increase CF_MAX_LEVELS\n");
1016 for (i = 0; i <
count; i++) {
1035 struct cf_setting *
sets;
1037 int error, i, set_count;
1039 dev = oidp->oid_arg1;
1040 sbuf_new(&sb, NULL, 128, SBUF_AUTOEXTEND);
1043 set_count = MAX_SETTINGS;
1049 error = CPUFREQ_DRV_SETTINGS(dev,
sets, &set_count);
1053 for (i = 0; i < set_count; i++)
1073 SYSCTL_ADD_CONST_STRING(&sc->sysctl_ctx,
1076 "cpufreq driver used by this cpu");
1083 device_t cf_dev, cpu_dev;
1089 OID_AUTO,
"freq_settings",
1090 CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_NEEDGIANT, dev, 0,
1100 sc->max_mhz = CPUFREQ_VAL_UNKNOWN;
1101 MPASS(sc->cf_drv_dev != NULL);
1106 cf_dev = BUS_ADD_CHILD(cpu_dev, 0,
"cpufreq",
device_get_unit(cpu_dev));
1116 sc->cf_drv_dev = dev;
1133 if (cf_dev == NULL) {
1135 "warning: cpufreq_unregister called with no cpufreq device active\n");
1139 MPASS(sc->cf_drv_dev == dev);
1149 EVENTHANDLER_INVOKE(cpufreq_levels_changed,
const struct cf_level * level
device_property_type_t type
static int cpufreq_detach(device_t dev)
DRIVER_MODULE(cpufreq, cpu, cpufreq_driver, cpufreq_dc, 0, 0)
static SYSCTL_NODE(_debug, OID_AUTO, cpufreq, CTLFLAG_RD|CTLFLAG_MPSAFE, NULL, "cpufreq debugging")
static int cpufreq_levels_sysctl(SYSCTL_HANDLER_ARGS)
static int cpufreq_attach(device_t dev)
static struct cf_level * cpufreq_dup_set(struct cpufreq_softc *sc, struct cf_level *dup, struct cf_setting *set)
SYSCTL_INT(_debug_cpufreq, OID_AUTO, lowest, CTLFLAG_RWTUN, &cf_lowest_freq, 1, "Don't provide levels below this frequency.")
int cpufreq_settings_changed(device_t dev)
static int cf_levels_method(device_t dev, struct cf_level *levels, int *count)
static int cpufreq_get_frequency(device_t dev)
static void cpufreq_add_freq_driver_sysctl(device_t cf_dev)
static int cf_set_method(device_t dev, const struct cf_level *level, int priority)
int cpufreq_register(device_t dev)
static int cpufreq_add_levels(device_t cf_dev, struct cf_setting_lst *rel_sets)
static void cpufreq_startup_task(void *ctx, int pending)
static int cf_lowest_freq
static int cpufreq_get_level(device_t dev, struct cf_level *levels, int count)
int cpufreq_unregister(device_t dev)
static int cpufreq_curr_sysctl(SYSCTL_HANDLER_ARGS)
static int cpufreq_settings_sysctl(SYSCTL_HANDLER_ARGS)
static driver_t cpufreq_driver
static int cpufreq_expand_set(struct cpufreq_softc *sc, struct cf_setting_array *set_arr)
static device_method_t cpufreq_methods[]
static devclass_t cpufreq_dc
static int cf_get_method(device_t dev, struct cf_level *level)
TAILQ_HEAD(cf_setting_lst, cf_setting_array)
static int cpufreq_insert_abs(struct cpufreq_softc *sc, struct cf_setting *sets, int count)
SLIST_HEAD(et_eventtimers_list, eventtimer)
void *() malloc(size_t size, struct malloc_type *mtp, int flags)
void free(void *addr, struct malloc_type *mtp)
int sysctl_ctx_free(struct sysctl_ctx_list *clist)
int sysctl_handle_int(SYSCTL_HANDLER_ARGS)
int sysctl_handle_string(SYSCTL_HANDLER_ARGS)
int sysctl_ctx_init(struct sysctl_ctx_list *c)
void sched_bind(struct thread *td, int cpu)
void sched_unbind(struct thread *td)
void sched_prio(struct thread *td, u_char prio)
struct cf_setting sets[MAX_SETTINGS]
struct cf_level curr_level
int devclass_get_devices(devclass_t dc, device_t **devlistp, int *devcountp)
Get a list of devices in the devclass.
int device_probe(device_t dev)
Probe a device, and return this status.
int device_detach(device_t dev)
Detach a driver from a device.
int device_is_attached(device_t dev)
Return non-zero if the device currently has a driver attached to it.
void device_quiet(device_t dev)
Set the DF_QUIET flag for the device.
struct sysctl_oid * device_get_sysctl_tree(device_t dev)
int device_printf(device_t dev, const char *fmt,...)
Print the name of the device followed by a colon, a space and the result of calling vprintf() with th...
device_t device_get_parent(device_t dev)
Return the parent of a device.
int device_get_unit(device_t dev)
Return the device's unit number.
device_t device_find_child(device_t dev, const char *classname, int unit)
Find a device given a unit number.
int device_delete_child(device_t dev, device_t child)
Delete a device.
struct sysctl_ctx_list * device_get_sysctl_ctx(device_t dev)
int device_attach(device_t dev)
Attach a device driver to a device.
int device_probe_and_attach(device_t dev)
Probe a device and attach a driver if possible.
int bus_generic_probe(device_t dev)
Helper function for implementing DEVICE_PROBE()
void * device_get_softc(device_t dev)
Return the device's softc field.
const char * device_get_nameunit(device_t dev)
Return a string containing the device's devclass name followed by an ascii representation of the devi...
int printf(const char *fmt,...)
int sbuf_finish(struct sbuf *s)
void sbuf_delete(struct sbuf *s)
int sbuf_printf(struct sbuf *s, const char *fmt,...)
ssize_t sbuf_len(struct sbuf *s)
char * sbuf_data(struct sbuf *s)
int sbuf_cpy(struct sbuf *s, const char *str)
struct sbuf * sbuf_new(struct sbuf *s, char *buf, int length, int flags)
int sbuf_trim(struct sbuf *s)
int taskqueue_enqueue(struct taskqueue *queue, struct task *task)