# list of DOCBOOKS.
DOCBOOKS := z8530book.xml \
- kernel-hacking.xml kernel-locking.xml deviceiobook.xml \
+ kernel-hacking.xml kernel-locking.xml \
writing_usb_driver.xml networking.xml \
kernel-api.xml filesystems.xml lsm.xml kgdb.xml \
gadget.xml libata.xml mtdnand.xml librs.xml rapidio.xml \
genericirq.xml s390-drivers.xml scsi.xml \
- sh.xml regulator.xml w1.xml \
+ sh.xml w1.xml \
writing_musb_glue_layer.xml
ifeq ($(DOCBOOKS),)
Then, you need a user space tool named aer-inject, which can be gotten
from:
- http://www.kernel.org/pub/linux/utils/pci/aer-inject/
+ https://git.kernel.org/cgit/linux/kernel/git/gong.chen/aer-inject.git/
More information about aer-inject can be found in the document comes
with its source code.
cpuidle.off=1 [CPU_IDLE]
disable the cpuidle sub-system
+ cpufreq.off=1 [CPU_FREQ]
+ disable the cpufreq sub-system
+
cpu_init_udelay=N
[X86] Delay for N microsec between assert and de-assert
of APIC INIT to start processors. This delay occurs
functions that can be changed at run time by the
set_graph_notrace file in the debugfs tracing directory.
+ ftrace_graph_max_depth=<uint>
+ [FTRACE] Used with the function graph tracer. This is
+ the max depth it will trace into a function. This value
+ can be changed at run time by the max_graph_depth file
+ in the tracefs tracing directory. default: 0 (no limit)
+
gamecon.map[2|3]=
[HW,JOY] Multisystem joystick and NES/SNES/PSX pad
support via parallel port (up to 5 devices per port)
| | | | |
| Qualcomm Tech. | Falkor v1 | E1003 | QCOM_FALKOR_ERRATUM_1003 |
| Qualcomm Tech. | Falkor v1 | E1009 | QCOM_FALKOR_ERRATUM_1009 |
+| Qualcomm Tech. | QDF2400 ITS | E0065 | QCOM_QDF2400_ERRATUM_0065 |
# If true, `todo` and `todoList` produce output, else they produce nothing.
todo_include_todos = False
-primary_domain = 'C'
+primary_domain = 'c'
highlight_language = 'none'
# -- Options for HTML output ----------------------------------------------
to collect more or less stable coverage that is function of syscall inputs.
To achieve this goal it does not collect coverage in soft/hard interrupts
and instrumentation of some inherently non-deterministic parts of kernel is
-disbled (e.g. scheduler, locking).
+disabled (e.g. scheduler, locking).
Usage
-----
- "hardkernel,odroid-c2" (Meson gxbb)
- "amlogic,p200" (Meson gxbb)
- "amlogic,p201" (Meson gxbb)
+ - "wetek,hub" (Meson gxbb)
+ - "wetek,play2" (Meson gxbb)
- "amlogic,p212" (Meson gxl s905x)
- "amlogic,p230" (Meson gxl s905d)
- "amlogic,p231" (Meson gxl s905d)
opp@1000000000 {
opp-hz = /bits/ 64 <1000000000>;
- opp-microvolt = <970000 975000 985000>;
+ opp-microvolt = <975000 970000 985000>;
opp-microamp = <70000>;
clock-latency-ns = <300000>;
opp-suspend;
};
opp@1100000000 {
opp-hz = /bits/ 64 <1100000000>;
- opp-microvolt = <980000 1000000 1010000>;
+ opp-microvolt = <1000000 980000 1010000>;
opp-microamp = <80000>;
clock-latency-ns = <310000>;
};
opp@1000000000 {
opp-hz = /bits/ 64 <1000000000>;
- opp-microvolt = <970000 975000 985000>;
+ opp-microvolt = <975000 970000 985000>;
opp-microamp = <70000>;
clock-latency-ns = <300000>;
opp-suspend;
};
opp@1100000000 {
opp-hz = /bits/ 64 <1100000000>;
- opp-microvolt = <980000 1000000 1010000>;
+ opp-microvolt = <1000000 980000 1010000>;
opp-microamp = <80000>;
clock-latency-ns = <310000>;
};
opp@1000000000 {
opp-hz = /bits/ 64 <1000000000>;
- opp-microvolt = <970000 975000 985000>;
+ opp-microvolt = <975000 970000 985000>;
opp-microamp = <70000>;
clock-latency-ns = <300000>;
opp-suspend;
};
opp@1100000000 {
opp-hz = /bits/ 64 <1100000000>;
- opp-microvolt = <980000 1000000 1010000>;
+ opp-microvolt = <1000000 980000 1010000>;
opp-microamp = <80000>;
clock-latency-ns = <310000>;
};
opp@1300000000 {
opp-hz = /bits/ 64 <1300000000>;
- opp-microvolt = <1045000 1050000 1055000>;
+ opp-microvolt = <1050000 1045000 1055000>;
opp-microamp = <95000>;
clock-latency-ns = <400000>;
opp-suspend;
};
opp@1500000000 {
opp-hz = /bits/ 64 <1500000000>;
- opp-microvolt = <1010000 1100000 1110000>;
+ opp-microvolt = <1100000 1010000 1110000>;
opp-microamp = <95000>;
clock-latency-ns = <400000>;
turbo-mode;
opp@1000000000 {
opp-hz = /bits/ 64 <1000000000>;
- opp-microvolt = <970000 975000 985000>, /* Supply 0 */
- <960000 965000 975000>, /* Supply 1 */
- <960000 965000 975000>; /* Supply 2 */
+ opp-microvolt = <975000 970000 985000>, /* Supply 0 */
+ <965000 960000 975000>, /* Supply 1 */
+ <965000 960000 975000>; /* Supply 2 */
opp-microamp = <70000>, /* Supply 0 */
<70000>, /* Supply 1 */
<70000>; /* Supply 2 */
opp@1000000000 {
opp-hz = /bits/ 64 <1000000000>;
- opp-microvolt = <970000 975000 985000>, /* Supply 0 */
- <960000 965000 975000>, /* Supply 1 */
- <960000 965000 975000>; /* Supply 2 */
+ opp-microvolt = <975000 970000 985000>, /* Supply 0 */
+ <965000 960000 975000>, /* Supply 1 */
+ <965000 960000 975000>; /* Supply 2 */
opp-microamp = <70000>, /* Supply 0 */
<0>, /* Supply 1 doesn't need this */
<70000>; /* Supply 2 */
*/
opp-supported-hw = <0xF 0xFFFFFFFF 0xFFFFFFFF>
opp-hz = /bits/ 64 <600000000>;
- opp-microvolt = <900000 915000 925000>;
+ opp-microvolt = <915000 900000 925000>;
...
};
*/
opp-supported-hw = <0x20 0xff0000ff 0x0000f4f0>
opp-hz = /bits/ 64 <800000000>;
- opp-microvolt = <900000 915000 925000>;
+ opp-microvolt = <915000 900000 925000>;
...
};
};
opp@1000000000 {
opp-hz = /bits/ 64 <1000000000>;
- opp-microvolt-slow = <900000 915000 925000>;
- opp-microvolt-fast = <970000 975000 985000>;
+ opp-microvolt-slow = <915000 900000 925000>;
+ opp-microvolt-fast = <975000 970000 985000>;
opp-microamp-slow = <70000>;
opp-microamp-fast = <71000>;
};
opp@1200000000 {
opp-hz = /bits/ 64 <1200000000>;
- opp-microvolt-slow = <900000 915000 925000>, /* Supply vcc0 */
- <910000 925000 935000>; /* Supply vcc1 */
- opp-microvolt-fast = <970000 975000 985000>, /* Supply vcc0 */
- <960000 965000 975000>; /* Supply vcc1 */
+ opp-microvolt-slow = <915000 900000 925000>, /* Supply vcc0 */
+ <925000 910000 935000>; /* Supply vcc1 */
+ opp-microvolt-fast = <975000 970000 985000>, /* Supply vcc0 */
+ <965000 960000 975000>; /* Supply vcc1 */
opp-microamp = <70000>; /* Will be used for both slow/fast */
};
};
Optional Properties:
- reg-names: In addition to the required properties, the following are optional
- "efuse-address" - Contains efuse base address used to pick up ABB info.
- - "ldo-address" - Contains address of ABB LDO overide register address.
+ - "ldo-address" - Contains address of ABB LDO override register.
"efuse-address" is required for this.
- ti,ldovbb-vset-mask - Required if ldo-address is set, mask for LDO override
register to provide override vset value.
- compatible : Should be "microchip,usb251xb" or one of the specific types:
"microchip,usb2512b", "microchip,usb2512bi", "microchip,usb2513b",
"microchip,usb2513bi", "microchip,usb2514b", "microchip,usb2514bi"
- - hub-reset-gpios : Should specify the gpio for hub reset
+ - reset-gpios : Should specify the gpio for hub reset
+ - reg : I2C address on the selected bus (default is <0x2C>)
Optional properties :
- - reg : I2C address on the selected bus (default is <0x2C>)
- skip-config : Skip Hub configuration, but only send the USB-Attach command
- - vendor-id : USB Vendor ID of the hub (16 bit, default is 0x0424)
- - product-id : USB Product ID of the hub (16 bit, default depends on type)
- - device-id : USB Device ID of the hub (16 bit, default is 0x0bb3)
- - language-id : USB Language ID (16 bit, default is 0x0000)
- - manufacturer : USB Manufacturer string (max 31 characters long)
- - product : USB Product string (max 31 characters long)
- - serial : USB Serial string (max 31 characters long)
+ - vendor-id : Set USB Vendor ID of the hub (16 bit, default is 0x0424)
+ - product-id : Set USB Product ID of the hub (16 bit, default depends on type)
+ - device-id : Set USB Device ID of the hub (16 bit, default is 0x0bb3)
+ - language-id : Set USB Language ID (16 bit, default is 0x0000)
+ - manufacturer : Set USB Manufacturer string (max 31 characters long)
+ - product : Set USB Product string (max 31 characters long)
+ - serial : Set USB Serial string (max 31 characters long)
- {bus,self}-powered : selects between self- and bus-powered operation (default
is self-powered)
- disable-hi-speed : disable USB Hi-Speed support
(default is individual)
- dynamic-power-switching : enable auto-switching from self- to bus-powered
operation if the local power source is removed or unavailable
- - oc-delay-{100us,4ms,8ms,16ms} : set over current timer delay (default is 8ms)
- - compound-device : indicated the hub is part of a compound device
+ - oc-delay-us : Delay time (in microseconds) for filtering the over-current
+ sense inputs. Valid values are 100, 4000, 8000 (default) and 16000. If
+ an invalid value is given, the default is used instead.
+ - compound-device : indicate the hub is part of a compound device
- port-mapping-mode : enable port mapping mode
- string-support : enable string descriptor support (required for manufacturer,
product and serial string configuration)
device connected.
- sp-disabled-ports : Specifies the ports which will be self-power disabled
- bp-disabled-ports : Specifies the ports which will be bus-power disabled
- - max-sp-power : Specifies the maximum current the hub consumes from an
- upstream port when operating as self-powered hub including the power
- consumption of a permanently attached peripheral if the hub is
- configured as a compound device. The value is given in mA in a 0 - 500
- range (default is 2).
- - max-bp-power : Specifies the maximum current the hub consumes from an
- upstream port when operating as bus-powered hub including the power
- consumption of a permanently attached peripheral if the hub is
- configured as a compound device. The value is given in mA in a 0 - 500
- range (default is 100).
- - max-sp-current : Specifies the maximum current the hub consumes from an
- upstream port when operating as self-powered hub EXCLUDING the power
- consumption of a permanently attached peripheral if the hub is
- configured as a compound device. The value is given in mA in a 0 - 500
- range (default is 2).
- - max-bp-current : Specifies the maximum current the hub consumes from an
- upstream port when operating as bus-powered hub EXCLUDING the power
- consumption of a permanently attached peripheral if the hub is
- configured as a compound device. The value is given in mA in a 0 - 500
- range (default is 100).
- - power-on-time : Specifies the time it takes from the time the host initiates
- the power-on sequence to a port until the port has adequate power. The
- value is given in ms in a 0 - 510 range (default is 100ms).
+ - power-on-time-ms : Specifies the time it takes from the time the host
+ initiates the power-on sequence to a port until the port has adequate
+ power. The value is given in ms in a 0 - 510 range (default is 100ms).
Examples:
usb2512b@2c {
compatible = "microchip,usb2512b";
- hub-reset-gpios = <&gpio1 4 GPIO_ACTIVE_LOW>;
+ reg = <0x2c>;
+ reset-gpios = <&gpio1 4 GPIO_ACTIVE_LOW>;
};
usb2514b@2c {
vivante Vivante Corporation
voipac Voipac Technologies s.r.o.
wd Western Digital Corp.
+wetek WeTek Electronics, limited.
wexler Wexler
winbond Winbond Electronics corp.
wlf Wolfson Microelectronics
int (*permission) (struct inode *, int, unsigned int);
int (*get_acl)(struct inode *, int);
int (*setattr) (struct dentry *, struct iattr *);
- int (*getattr) (struct vfsmount *, struct dentry *, struct kstat *);
+ int (*getattr) (const struct path *, struct dentry *, struct kstat *,
+ u32, unsigned int);
ssize_t (*listxattr) (struct dentry *, char *, size_t);
int (*fiemap)(struct inode *, struct fiemap_extent_info *, u64 start, u64 len);
void (*update_time)(struct inode *, struct timespec *, int);
int (*permission) (struct inode *, int);
int (*get_acl)(struct inode *, int);
int (*setattr) (struct dentry *, struct iattr *);
- int (*getattr) (struct vfsmount *mnt, struct dentry *, struct kstat *);
+ int (*getattr) (const struct path *, struct dentry *, struct kstat *,
+ u32, unsigned int);
ssize_t (*listxattr) (struct dentry *, char *, size_t);
void (*update_time)(struct inode *, struct timespec *, int);
int (*atomic_open)(struct inode *, struct dentry *, struct file *,
protection faults (so-called "kernel oops").
If you run into some kind of deadlock, you can try to dump a call trace
-for each process using sysrq-t (see Documentation/sysrq.txt).
+for each process using sysrq-t (see Documentation/admin-guide/sysrq.rst).
This way it is possible to figure where *exactly* some process in "D"
state is stuck.
under the device's power directory.
Notification mechanisms:
-The per-device PM QoS framework has 2 different and distinct notification trees:
-a per-device notification tree and a global notification tree.
+The per-device PM QoS framework has a per-device notification tree.
int dev_pm_qos_add_notifier(device, notifier):
Adds a notification callback function for the device.
int dev_pm_qos_remove_notifier(device, notifier):
Removes the notification callback function for the device.
-int dev_pm_qos_add_global_notifier(notifier):
-Adds a notification callback function in the global notification tree of the
-framework.
-The callback is called when the aggregated value for any device is changed
-(for resume latency device PM QoS only).
-
-int dev_pm_qos_remove_global_notifier(notifier):
-Removes the notification callback function from the global notification tree
-of the framework.
-
Active state latency tolerance
* If the suspend callback returns an error code different from -EBUSY and
-EAGAIN, the PM core regards this as a fatal error and will refuse to run
the helper functions described in Section 4 for the device until its status
- is directly set to either'active', or 'suspended' (the PM core provides
+ is directly set to either 'active', or 'suspended' (the PM core provides
special helper functions for this purpose).
In particular, if the driver requires remote wakeup capability (i.e. hardware
one to complete
spinlock_t lock;
- - lock used for synchronisation
+ - lock used for synchronization
atomic_t usage_count;
- the usage counter of the device
probe. This can happen with systems such as the network device layer.
It may be desirable to suspend the device once ->probe() has finished.
-Therefore the driver core uses the asyncronous pm_request_idle() to submit a
+Therefore the driver core uses the asynchronous pm_request_idle() to submit a
request to execute the subsystem-level idle callback for the device at that
time. A driver that makes use of the runtime autosuspend feature, may want to
update the last busy mark before returning from ->probe().
This is particularly useful for syncing disks unmounting & rebooting
if the machine gets partially hung.
-Read Documentation/sysrq.txt for more info
+Read Documentation/admin-guide/sysrq.rst for more info
References:
===========
usage. This is called key->payload.rcu_data0. The following accessors
wrap the RCU calls to this element:
- rcu_assign_keypointer(struct key *key, void *data);
- void *rcu_dereference_key(struct key *key);
+ (a) Set or change the first payload pointer:
+
+ rcu_assign_keypointer(struct key *key, void *data);
+
+ (b) Read the first payload pointer with the key semaphore held:
+
+ [const] void *dereference_key_locked([const] struct key *key);
+
+ Note that the return value will inherit its constness from the key
+ parameter. Static analysis will give an error if it things the lock
+ isn't held.
+
+ (c) Read the first payload pointer with the RCU read lock held:
+
+ const void *dereference_key_rcu(const struct key *key);
===================
- softlockup_all_cpu_backtrace
- soft_watchdog
- stop-a [ SPARC only ]
-- sysrq ==> Documentation/sysrq.txt
+- sysrq ==> Documentation/admin-guide/sysrq.rst
- sysctl_writes_strict
- tainted
- threads-max
functions). Unlike the Tracepoint based event, this can be added and removed
dynamically, on the fly.
-To enable this feature, build your kernel with CONFIG_KPROBE_EVENT=y.
+To enable this feature, build your kernel with CONFIG_KPROBE_EVENTS=y.
Similar to the events tracer, this doesn't need to be activated via
current_tracer. Instead of that, add probe points via
Overview
--------
Uprobe based trace events are similar to kprobe based trace events.
-To enable this feature, build your kernel with CONFIG_UPROBE_EVENT=y.
+To enable this feature, build your kernel with CONFIG_UPROBE_EVENTS=y.
Similar to the kprobe-event tracer, this doesn't need to be activated via
current_tracer. Instead of that, add probe points via
컨트롤 의존성
-------------
+현재의 컴파일러들은 컨트롤 의존성을 이해하고 있지 않기 때문에 컨트롤 의존성은
+약간 다루기 어려울 수 있습니다. 이 섹션의 목적은 여러분이 컴파일러의 무시로
+인해 여러분의 코드가 망가지는 걸 막을 수 있도록 돕는겁니다.
+
로드-로드 컨트롤 의존성은 데이터 의존성 배리어만으로는 정확히 동작할 수가
없어서 읽기 메모리 배리어를 필요로 합니다. 아래의 코드를 봅시다:
q = READ_ONCE(a);
if (q) {
- WRITE_ONCE(b, p);
+ WRITE_ONCE(b, 1);
}
컨트롤 의존성은 보통 다른 타입의 배리어들과 짝을 맞춰 사용됩니다. 그렇다곤
-하나, READ_ONCE() 는 반드시 사용해야 함을 부디 명심하세요! READ_ONCE() 가
-없다면, 컴파일러가 'a' 로부터의 로드를 'a' 로부터의 또다른 로드와, 'b' 로의
-스토어를 'b' 로의 또다른 스토어와 조합해 버려 매우 비직관적인 결과를 초래할 수
-있습니다.
+하나, READ_ONCE() 도 WRITE_ONCE() 도 선택사항이 아니라 필수사항임을 부디
+명심하세요! READ_ONCE() 가 없다면, 컴파일러는 'a' 로부터의 로드를 'a' 로부터의
+또다른 로드와 조합할 수 있습니다. WRITE_ONCE() 가 없다면, 컴파일러는 'b' 로의
+스토어를 'b' 로의 또라느 스토어들과 조합할 수 있습니다. 두 경우 모두 순서에
+있어 상당히 비직관적인 결과를 초래할 수 있습니다.
이걸로 끝이 아닌게, 컴파일러가 변수 'a' 의 값이 항상 0이 아니라고 증명할 수
있다면, 앞의 예에서 "if" 문을 없애서 다음과 같이 최적화 할 수도 있습니다:
q = a;
- b = p; /* BUG: Compiler and CPU can both reorder!!! */
+ b = 1; /* BUG: Compiler and CPU can both reorder!!! */
그러니 READ_ONCE() 를 반드시 사용하세요.
q = READ_ONCE(a);
if (q) {
barrier();
- WRITE_ONCE(b, p);
+ WRITE_ONCE(b, 1);
do_something();
} else {
barrier();
- WRITE_ONCE(b, p);
+ WRITE_ONCE(b, 1);
do_something_else();
}
q = READ_ONCE(a);
barrier();
- WRITE_ONCE(b, p); /* BUG: No ordering vs. load from a!!! */
+ WRITE_ONCE(b, 1); /* BUG: No ordering vs. load from a!!! */
if (q) {
- /* WRITE_ONCE(b, p); -- moved up, BUG!!! */
+ /* WRITE_ONCE(b, 1); -- moved up, BUG!!! */
do_something();
} else {
- /* WRITE_ONCE(b, p); -- moved up, BUG!!! */
+ /* WRITE_ONCE(b, 1); -- moved up, BUG!!! */
do_something_else();
}
q = READ_ONCE(a);
if (q) {
- smp_store_release(&b, p);
+ smp_store_release(&b, 1);
do_something();
} else {
- smp_store_release(&b, p);
+ smp_store_release(&b, 1);
do_something_else();
}
q = READ_ONCE(a);
if (q) {
- WRITE_ONCE(b, p);
+ WRITE_ONCE(b, 1);
do_something();
} else {
- WRITE_ONCE(b, r);
+ WRITE_ONCE(b, 2);
do_something_else();
}
q = READ_ONCE(a);
if (q % MAX) {
- WRITE_ONCE(b, p);
+ WRITE_ONCE(b, 1);
do_something();
} else {
- WRITE_ONCE(b, r);
+ WRITE_ONCE(b, 2);
do_something_else();
}
위의 코드를 아래와 같이 바꿔버릴 수 있습니다:
q = READ_ONCE(a);
- WRITE_ONCE(b, p);
+ WRITE_ONCE(b, 1);
do_something_else();
이렇게 되면, CPU 는 변수 'a' 로부터의 로드와 변수 'b' 로의 스토어 사이의 순서를
q = READ_ONCE(a);
BUILD_BUG_ON(MAX <= 1); /* Order load from a with store to b. */
if (q % MAX) {
- WRITE_ONCE(b, p);
+ WRITE_ONCE(b, 1);
do_something();
} else {
- WRITE_ONCE(b, r);
+ WRITE_ONCE(b, 2);
do_something_else();
}
q = READ_ONCE(a);
if (q) {
- WRITE_ONCE(b, p);
+ WRITE_ONCE(b, 1);
} else {
- WRITE_ONCE(b, r);
+ WRITE_ONCE(b, 2);
}
- WRITE_ONCE(c, 1); /* BUG: No ordering against the read from "a". */
+ WRITE_ONCE(c, 1); /* BUG: No ordering against the read from 'a'. */
-컴파일러는 volatile 타입에 대한 액세스를 재배치 할 수 없고 이 조건 하의 "b"
+컴파일러는 volatile 타입에 대한 액세스를 재배치 할 수 없고 이 조건 하의 'b'
로의 쓰기를 재배치 할 수 없기 때문에 여기에 순서 규칙이 존재한다고 주장하고
싶을 겁니다. 불행히도 이 경우에, 컴파일러는 다음의 가상의 pseudo-assembly 언어
-코드처럼 "b" 로의 두개의 쓰기 오퍼레이션을 conditional-move 인스트럭션으로
+코드처럼 'b' 로의 두개의 쓰기 오퍼레이션을 conditional-move 인스트럭션으로
번역할 수 있습니다:
ld r1,a
- ld r2,p
- ld r3,r
cmp r1,$0
- cmov,ne r4,r2
- cmov,eq r4,r3
+ cmov,ne r4,$1
+ cmov,eq r4,$2
st r4,b
st $1,c
-완화된 순서 규칙의 CPU 는 "a" 로부터의 로드와 "c" 로의 스토어 사이에 어떤
+완화된 순서 규칙의 CPU 는 'a' 로부터의 로드와 'c' 로의 스토어 사이에 어떤
종류의 의존성도 갖지 않을 겁니다. 이 컨트롤 의존성은 두개의 cmov 인스트럭션과
거기에 의존하는 스토어 에게만 적용될 겁니다. 짧게 말하자면, 컨트롤 의존성은
주어진 if 문의 then 절과 else 절에게만 (그리고 이 두 절 내에서 호출되는
함수들에게까지) 적용되지, 이 if 문을 뒤따르는 코드에는 적용되지 않습니다.
마지막으로, 컨트롤 의존성은 이행성 (transitivity) 을 제공하지 -않습니다-. 이건
-x 와 y 가 둘 다 0 이라는 초기값을 가졌다는 가정 하의 두개의 예제로
+'x' 와 'y' 가 둘 다 0 이라는 초기값을 가졌다는 가정 하의 두개의 예제로
보이겠습니다:
CPU 0 CPU 1
(*) 컨트롤 의존성은 이행성을 제공하지 -않습니다-. 이행성이 필요하다면,
smp_mb() 를 사용하세요.
+ (*) 컴파일러는 컨트롤 의존성을 이해하고 있지 않습니다. 따라서 컴파일러가
+ 여러분의 코드를 망가뜨리지 않도록 하는건 여러분이 해야 하는 일입니다.
+
SMP 배리어 짝맞추기
--------------------
slot. When changing an existing slot, it may be moved in the guest
physical memory space, or its flags may be modified. It may not be
resized. Slots may not overlap in guest physical address space.
+Bits 0-15 of "slot" specifies the slot id and this value should be
+less than the maximum number of user memory slots supported per VM.
+The maximum allowed slots can be queried using KVM_CAP_NR_MEMSLOTS,
+if this capability is supported by the architecture.
If KVM_CAP_MULTI_ADDRESS_SPACE is available, bits 16-31 of "slot"
specifies the address space which is being modified. They must be
This takes one argument, which is a single letter. It calls the
generic kernel's SysRq driver, which does whatever is called for by
- that argument. See the SysRq documentation in Documentation/sysrq.txt
- in your favorite kernel tree to see what letters are valid and what
- they do.
+ that argument. See the SysRq documentation in
+ Documentation/admin-guide/sysrq.rst in your favorite kernel tree to
+ see what letters are valid and what they do.
manager has to explicitly enable these events by setting appropriate
bits in uffdio_api.features passed to UFFDIO_API ioctl:
-UFFD_FEATURE_EVENT_EXIT - enable notification about exit() of the
-non-cooperative process. When the monitored process exits, the uffd
-manager will get UFFD_EVENT_EXIT.
-
UFFD_FEATURE_EVENT_FORK - enable userfaultfd hooks for fork(). When
this feature is enabled, the userfaultfd context of the parent process
is duplicated into the newly created process. The manager receives
FBTFT Framebuffer drivers
M: Thomas Petazzoni <thomas.petazzoni@free-electrons.com>
-M: Noralf Trønnes <noralf@tronnes.org>
S: Maintained
F: drivers/staging/fbtft/
F: include/uapi/linux/hsi/
HSO 3G MODEM DRIVER
-M: Jan Dumon <j.dumon@option.com>
-W: http://www.pharscape.org
-S: Maintained
+L: linux-usb@vger.kernel.org
+S: Orphan
F: drivers/net/usb/hso.c
HSR NETWORK PROTOCOL
L: linux-usb@vger.kernel.org
S: Maintained
F: drivers/usb/misc/usb251xb.c
-F: include/linux/platform_data/usb251xb.h
F: Documentation/devicetree/bindings/usb/usb251xb.txt
MICROSOFT SURFACE PRO 3 BUTTON DRIVER
S: Supported
F: drivers/scsi/qedi/
+QLOGIC QL41xxx FCOE DRIVER
+M: QLogic-Storage-Upstream@cavium.com
+L: linux-scsi@vger.kernel.org
+S: Supported
+F: drivers/scsi/qedf/
+
QNX4 FILESYSTEM
M: Anders Larsen <al@alarsen.net>
W: http://www.alarsen.net/linux/qnx4fs/
VERSION = 4
-PATCHLEVEL = 10
+PATCHLEVEL = 11
SUBLEVEL = 0
-EXTRAVERSION =
+EXTRAVERSION = -rc2
NAME = Fearless Coyote
# *DOCUMENTATION*
#ifdef __KERNEL__
#include <linux/user.h>
+#include <linux/mm_types.h>
/*
* Fill in the user structure for an ECOFF core dump.
* Copyright (C) 1996, Linus Torvalds
*/
+#include <linux/mm_types.h>
+
#include <asm/machvec.h>
#include <asm/compiler.h>
#include <asm-generic/mm_hooks.h>
*/
#include <linux/errno.h>
-#include <linux/sched.h>
+#include <linux/sched/signal.h>
+#include <linux/sched/mm.h>
+#include <linux/sched/task_stack.h>
+#include <linux/sched/cputime.h>
#include <linux/kernel.h>
#include <linux/mm.h>
#include <linux/smp.h>
#include <linux/errno.h>
#include <linux/module.h>
#include <linux/sched.h>
+#include <linux/sched/debug.h>
+#include <linux/sched/task.h>
+#include <linux/sched/task_stack.h>
#include <linux/kernel.h>
#include <linux/mm.h>
#include <linux/smp.h>
#include <linux/kernel.h>
#include <linux/sched.h>
+#include <linux/sched/task_stack.h>
#include <linux/mm.h>
#include <linux/smp.h>
#include <linux/errno.h>
* 1997-11-02 Modified for POSIX.1b signals by Richard Henderson
*/
-#include <linux/sched.h>
+#include <linux/sched/signal.h>
+#include <linux/sched/task_stack.h>
#include <linux/kernel.h>
#include <linux/signal.h>
#include <linux/errno.h>
#include <linux/kernel.h>
#include <linux/kernel_stat.h>
#include <linux/module.h>
-#include <linux/sched.h>
+#include <linux/sched/mm.h>
#include <linux/mm.h>
#include <linux/err.h>
#include <linux/threads.h>
#include <linux/jiffies.h>
#include <linux/mm.h>
-#include <linux/sched.h>
+#include <linux/sched/signal.h>
+#include <linux/sched/debug.h>
#include <linux/tty.h>
#include <linux/delay.h>
#include <linux/extable.h>
#include <linux/types.h>
#include <linux/kernel.h>
#include <linux/sched.h>
+#include <asm/ptrace.h>
#include <linux/uaccess.h>
* Copyright (C) 1995 Linus Torvalds
*/
-#include <linux/sched.h>
+#include <linux/sched/signal.h>
#include <linux/kernel.h>
#include <linux/mm.h>
#include <asm/io.h>
#define _ASM_ARC_HUGEPAGE_H
#include <linux/types.h>
+#define __ARCH_USE_5LEVEL_HACK
#include <asm-generic/pgtable-nopmd.h>
static inline pte_t pmd_pte(pmd_t pmd)
#include <asm/arcregs.h>
#include <asm/tlb.h>
+#include <linux/sched/mm.h>
#include <asm-generic/mm_hooks.h>
#include <asm/page.h>
#include <asm/mmu.h>
+#define __ARCH_USE_5LEVEL_HACK
#include <asm-generic/pgtable-nopmd.h>
#include <linux/const.h>
#include <asm/asm-offsets.h>
#include <linux/sched.h>
+#include <linux/sched/debug.h>
#ifdef CONFIG_ARC_PLAT_EZNPS
#include <plat/ctop.h>
#endif
#include <linux/kgdb.h>
#include <linux/sched.h>
+#include <linux/sched/task_stack.h>
#include <asm/disasm.h>
#include <asm/cacheflush.h>
#include <linux/errno.h>
#include <linux/module.h>
#include <linux/sched.h>
+#include <linux/sched/task.h>
+#include <linux/sched/task_stack.h>
+
#include <linux/mm.h>
#include <linux/fs.h>
#include <linux/unistd.h>
#include <linux/ptrace.h>
#include <linux/tracehook.h>
+#include <linux/sched/task_stack.h>
#include <linux/regset.h>
#include <linux/unistd.h>
#include <linux/elf.h>
#include <linux/uaccess.h>
#include <linux/syscalls.h>
#include <linux/tracehook.h>
+#include <linux/sched/task_stack.h>
+
#include <asm/ucontext.h>
struct rt_sigframe {
*/
#include <linux/spinlock.h>
-#include <linux/sched.h>
+#include <linux/sched/mm.h>
#include <linux/interrupt.h>
#include <linux/profile.h>
#include <linux/mm.h>
#include <linux/export.h>
#include <linux/stacktrace.h>
#include <linux/kallsyms.h>
+#include <linux/sched/debug.h>
+
#include <asm/arcregs.h>
#include <asm/unwind.h>
#include <asm/switch_to.h>
* Rahul Trivedi: Codito Technologies 2004
*/
-#include <linux/sched.h>
+#include <linux/sched/signal.h>
#include <linux/kdebug.h>
#include <linux/uaccess.h>
#include <linux/ptrace.h>
#include <linux/fs_struct.h>
#include <linux/proc_fs.h>
#include <linux/file.h>
+#include <linux/sched/mm.h>
+#include <linux/sched/debug.h>
+
#include <asm/arcregs.h>
#include <asm/irqflags.h>
#include <linux/signal.h>
#include <linux/interrupt.h>
-#include <linux/sched.h>
+#include <linux/sched/signal.h>
#include <linux/errno.h>
#include <linux/ptrace.h>
#include <linux/uaccess.h>
#include <linux/fs.h>
#include <linux/mm.h>
#include <linux/mman.h>
-#include <linux/sched.h>
+#include <linux/sched/mm.h>
+
#include <asm/cacheflush.h>
#define COLOUR_ALIGN(addr, pgoff) \
#include <linux/module.h>
#include <linux/bug.h>
+#include <linux/mm_types.h>
+
#include <asm/arcregs.h>
#include <asm/setup.h>
#include <asm/mmu_context.h>
#include <linux/init.h>
#include <linux/kernel.h>
#include <linux/module.h>
-#include <linux/sched.h>
+#include <linux/sched/signal.h>
+#include <uapi/linux/sched/types.h>
#include <linux/interrupt.h>
#include <linux/cpu_pm.h>
#include <linux/cpu.h>
# CONFIG_LBDAF is not set
# CONFIG_BLK_DEV_BSG is not set
# CONFIG_IOSCHED_DEADLINE is not set
-CONFIG_ARCH_MULTI_V4T=y
+CONFIG_ARCH_MULTI_V4=y
# CONFIG_ARCH_MULTI_V7 is not set
-CONFIG_KEYBOARD_GPIO_POLLED=y
CONFIG_ARCH_MOXART=y
CONFIG_MACH_UC7112LX=y
CONFIG_PREEMPT=y
CONFIG_SERIAL_8250_SHARE_IRQ=y
CONFIG_SERIAL_OF_PLATFORM=y
# CONFIG_HW_RANDOM is not set
-CONFIG_DEBUG_GPIO=y
-CONFIG_GPIO_SYSFS=y
CONFIG_GPIO_MOXART=y
-CONFIG_POWER_SUPPLY=y
CONFIG_POWER_RESET=y
CONFIG_POWER_RESET_GPIO=y
+CONFIG_POWER_SUPPLY=y
# CONFIG_HWMON is not set
CONFIG_WATCHDOG=y
CONFIG_WATCHDOG_CORE=y
CONFIG_MOXART_WDT=y
# CONFIG_USB_SUPPORT is not set
CONFIG_MMC=y
-CONFIG_MMC_SDHCI_MOXART=y
+CONFIG_MMC_SDHCI=y
+CONFIG_MMC_SDHCI_PLTFM=y
+CONFIG_MMC_MOXART=y
CONFIG_NEW_LEDS=y
CONFIG_LEDS_CLASS=y
CONFIG_LEDS_GPIO=y
+CONFIG_LEDS_TRIGGERS=y
CONFIG_LEDS_TRIGGER_TIMER=y
CONFIG_LEDS_TRIGGER_ONESHOT=y
CONFIG_LEDS_TRIGGER_HEARTBEAT=y
ce-obj-$(CONFIG_CRYPTO_SHA2_ARM_CE) += sha2-arm-ce.o
ce-obj-$(CONFIG_CRYPTO_GHASH_ARM_CE) += ghash-arm-ce.o
ce-obj-$(CONFIG_CRYPTO_CRCT10DIF_ARM_CE) += crct10dif-arm-ce.o
-ce-obj-$(CONFIG_CRYPTO_CRC32_ARM_CE) += crc32-arm-ce.o
+crc-obj-$(CONFIG_CRYPTO_CRC32_ARM_CE) += crc32-arm-ce.o
+
+ifneq ($(crc-obj-y)$(crc-obj-m),)
+ifeq ($(call as-instr,.arch armv8-a\n.arch_extension crc,y,n),y)
+ce-obj-y += $(crc-obj-y)
+ce-obj-m += $(crc-obj-m)
+else
+$(warning These CRC Extensions modules need binutils 2.23 or higher)
+$(warning $(crc-obj-y) $(crc-obj-m))
+endif
+endif
ifneq ($(ce-obj-y)$(ce-obj-m),)
ifeq ($(call as-instr,.fpu crypto-neon-fp-armv8,y,n),y)
vld1.8 {q3-q4}, [BUF, :128]!
vmov.i8 qzr, #0
vmov.i8 qCONSTANT, #0
- vmov dCONSTANTl[0], CRC
+ vmov.32 dCONSTANTl[0], CRC
veor.8 d2, d2, dCONSTANTl
sub LEN, LEN, #0x40
cmp LEN, #0x40
#define HSR_EC_IABT_HYP (0x21)
#define HSR_EC_DABT (0x24)
#define HSR_EC_DABT_HYP (0x25)
+#define HSR_EC_MAX (0x3f)
#define HSR_WFI_IS_WFE (_AC(1, UL) << 0)
#define __KVM_HAVE_ARCH_INTC_INITIALIZED
#define KVM_USER_MEM_SLOTS 32
-#define KVM_PRIVATE_MEM_SLOTS 4
#define KVM_COALESCED_MMIO_PAGE_OFFSET 1
#define KVM_HAVE_ONE_REG
#define KVM_HALT_POLL_NS_DEFAULT 500000
#include <linux/compiler.h>
#include <linux/sched.h>
+#include <linux/mm_types.h>
#include <linux/preempt.h>
+
#include <asm/cacheflush.h>
#include <asm/cachetype.h>
#include <asm/proc-fns.h>
#else
+#define __ARCH_USE_5LEVEL_HACK
#include <asm-generic/pgtable-nopud.h>
#include <asm/memory.h>
#include <asm/pgtable-hwdef.h>
#ifndef _ASMARM_TLBFLUSH_H
#define _ASMARM_TLBFLUSH_H
+#ifndef __ASSEMBLY__
+# include <linux/mm_types.h>
+#endif
+
#ifdef CONFIG_MMU
#include <asm/glue.h>
#elif defined(CONFIG_SMP) /* !CONFIG_MMU */
#ifndef __ASSEMBLY__
-
-#include <linux/mm_types.h>
-
static inline void local_flush_tlb_all(void) { }
static inline void local_flush_tlb_mm(struct mm_struct *mm) { }
static inline void local_flush_tlb_page(struct vm_area_struct *vma, unsigned long uaddr) { }
#include <linux/kernel.h>
#include <linux/perf_event.h>
#include <linux/bug.h>
+#include <linux/sched/task_stack.h>
#include <asm/perf_regs.h>
#include <asm/ptrace.h>
#include <linux/export.h>
#include <linux/sched.h>
+#include <linux/sched/debug.h>
+#include <linux/sched/task.h>
+#include <linux/sched/task_stack.h>
#include <linux/kernel.h>
#include <linux/mm.h>
#include <linux/stddef.h>
* published by the Free Software Foundation.
*/
#include <linux/kernel.h>
-#include <linux/sched.h>
+#include <linux/sched/signal.h>
+#include <linux/sched/task_stack.h>
#include <linux/mm.h>
#include <linux/elf.h>
#include <linux/smp.h>
#include <linux/delay.h>
#include <linux/init.h>
#include <linux/spinlock.h>
-#include <linux/sched.h>
+#include <linux/sched/mm.h>
+#include <linux/sched/hotplug.h>
+#include <linux/sched/task_stack.h>
#include <linux/interrupt.h>
#include <linux/cache.h>
#include <linux/profile.h>
#include <linux/export.h>
#include <linux/sched.h>
+#include <linux/sched/debug.h>
#include <linux/stacktrace.h>
#include <asm/stacktrace.h>
#include <linux/init.h>
#include <linux/slab.h>
+#include <linux/mm_types.h>
#include <asm/cacheflush.h>
#include <asm/idmap.h>
#include <linux/proc_fs.h>
#include <linux/seq_file.h>
#include <linux/sched.h>
+#include <linux/sched/mm.h>
#include <linux/syscalls.h>
#include <linux/perf_event.h>
#include <linux/syscalls.h>
#include <linux/errno.h>
#include <linux/fs.h>
+#include <linux/cred.h>
#include <linux/fcntl.h>
#include <linux/eventpoll.h>
#include <linux/sem.h>
#include <linux/nodemask.h>
#include <linux/of.h>
#include <linux/sched.h>
+#include <linux/sched/topology.h>
#include <linux/slab.h>
#include <linux/string.h>
#include <linux/bug.h>
#include <linux/delay.h>
#include <linux/init.h>
-#include <linux/sched.h>
+#include <linux/sched/signal.h>
+#include <linux/sched/debug.h>
+#include <linux/sched/task_stack.h>
#include <linux/irq.h>
#include <linux/atomic.h>
case KVM_CAP_MAX_VCPUS:
r = KVM_MAX_VCPUS;
break;
+ case KVM_CAP_NR_MEMSLOTS:
+ r = KVM_USER_MEM_SLOTS;
+ break;
case KVM_CAP_MSI_DEVID:
if (!kvm)
r = -EINVAL;
return 1;
}
+static int kvm_handle_unknown_ec(struct kvm_vcpu *vcpu, struct kvm_run *run)
+{
+ u32 hsr = kvm_vcpu_get_hsr(vcpu);
+
+ kvm_pr_unimpl("Unknown exception class: hsr: %#08x\n",
+ hsr);
+
+ kvm_inject_undefined(vcpu);
+ return 1;
+}
+
static exit_handle_fn arm_exit_handlers[] = {
+ [0 ... HSR_EC_MAX] = kvm_handle_unknown_ec,
[HSR_EC_WFI] = kvm_handle_wfx,
[HSR_EC_CP15_32] = kvm_handle_cp15_32,
[HSR_EC_CP15_64] = kvm_handle_cp15_64,
{
u8 hsr_ec = kvm_vcpu_trap_get_class(vcpu);
- if (hsr_ec >= ARRAY_SIZE(arm_exit_handlers) ||
- !arm_exit_handlers[hsr_ec]) {
- kvm_err("Unknown exception class: hsr: %#08x\n",
- (unsigned int)kvm_vcpu_get_hsr(vcpu));
- BUG();
- }
-
return arm_exit_handlers[hsr_ec];
}
#include <linux/of.h>
#include <linux/of_address.h>
#include <linux/sched.h>
+#include <linux/sched/clock.h>
#include <linux/smp.h>
#include <asm/cacheflush.h>
#include <linux/kernel.h>
#include <linux/sched.h>
+#include <linux/sched/clock.h>
#include <linux/clk.h>
#include <linux/err.h>
#include <linux/io.h>
#include <linux/kernel.h>
#include <linux/types.h>
#include <linux/sched.h>
+#include <linux/sched/mm.h>
#include <linux/interrupt.h>
#include <linux/completion.h>
#include <linux/reboot.h>
obj-y := pm.o
obj-$(CONFIG_UX500_SOC_DB8500) += cpu-db8500.o
obj-$(CONFIG_SMP) += platsmp.o
-obj-$(CONFIG_HOTPLUG_CPU) += hotplug.o
obj-$(CONFIG_PM_GENERIC_DOMAINS) += pm_domains.o
-
-CFLAGS_hotplug.o += -march=armv7-a
#include <asm/mach/map.h>
#include <asm/mach/arch.h>
-#include "setup.h"
-
#include "db8500-regs.h"
static int __init ux500_l2x0_unlock(void)
+++ /dev/null
-/*
- * Copyright (C) STMicroelectronics 2009
- * Copyright (C) ST-Ericsson SA 2010
- *
- * License Terms: GNU General Public License v2
- * Based on ARM realview platform
- *
- * Author: Sundar Iyer <sundar.iyer@stericsson.com>
- *
- */
-#include <linux/kernel.h>
-#include <linux/errno.h>
-#include <linux/smp.h>
-
-#include <asm/smp_plat.h>
-
-#include "setup.h"
-
-/*
- * platform-specific code to shutdown a CPU
- *
- * Called with IRQs disabled
- */
-void ux500_cpu_die(unsigned int cpu)
-{
- /* directly enter low power state, skipping secure registers */
- for (;;) {
- __asm__ __volatile__("dsb\n\t" "wfi\n\t"
- : : : "memory");
- if (pen_release == cpu_logical_map(cpu)) {
- /*
- * OK, proper wakeup, we're done
- */
- break;
- }
- }
-}
#include <asm/smp_plat.h>
#include <asm/smp_scu.h>
-#include "setup.h"
-
#include "db8500-regs.h"
/* Magic triggers in backup RAM */
return 0;
}
+#ifdef CONFIG_HOTPLUG_CPU
+void ux500_cpu_die(unsigned int cpu)
+{
+ wfi();
+}
+#endif
+
static const struct smp_operations ux500_smp_ops __initconst = {
.smp_prepare_cpus = ux500_smp_prepare_cpus,
.smp_boot_secondary = ux500_boot_secondary,
+++ /dev/null
-/*
- * Copyright (C) 2009 ST-Ericsson.
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
- *
- * These symbols are needed for board-specific files to call their
- * own cpu-specific files
- */
-#ifndef __ASM_ARCH_SETUP_H
-#define __ASM_ARCH_SETUP_H
-
-extern void ux500_cpu_die(unsigned int cpu);
-
-#endif /* __ASM_ARCH_SETUP_H */
#include <linux/moduleparam.h>
#include <linux/compiler.h>
#include <linux/kernel.h>
+#include <linux/sched/debug.h>
#include <linux/errno.h>
#include <linux/string.h>
#include <linux/proc_fs.h>
#include <linux/seq_file.h>
#include <linux/init.h>
-#include <linux/sched.h>
+#include <linux/sched/signal.h>
#include <linux/uaccess.h>
#include <asm/cp15.h>
#include <linux/kprobes.h>
#include <linux/uaccess.h>
#include <linux/page-flags.h>
-#include <linux/sched.h>
+#include <linux/sched/signal.h>
+#include <linux/sched/debug.h>
#include <linux/highmem.h>
#include <linux/perf_event.h>
#include <linux/module.h>
#include <linux/kernel.h>
#include <linux/slab.h>
+#include <linux/mm_types.h>
#include <asm/cputype.h>
#include <asm/idmap.h>
#include <linux/init.h>
#include <linux/bootmem.h>
#include <linux/mman.h>
+#include <linux/sched/signal.h>
+#include <linux/sched/task.h>
#include <linux/export.h>
#include <linux/nodemask.h>
#include <linux/initrd.h>
#include <linux/mm.h>
#include <linux/mman.h>
#include <linux/shm.h>
-#include <linux/sched.h>
+#include <linux/sched/signal.h>
+#include <linux/sched/mm.h>
#include <linux/io.h>
#include <linux/personality.h>
#include <linux/random.h>
#include <linux/types.h>
#include <linux/kernel.h>
#include <linux/signal.h>
-#include <linux/sched.h>
+#include <linux/sched/signal.h>
#include <linux/init.h>
#include <asm/thread_notify.h>
#include <linux/module.h>
#include <linux/slab.h>
#include <linux/stop_machine.h>
+#include <linux/sched/debug.h>
#include <linux/stringify.h>
#include <asm/traps.h>
#include <asm/opcodes.h>
#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/slab.h>
+#include <linux/sched/clock.h>
#include <linux/kprobes.h>
#include <linux/errno.h>
#include <linux/stddef.h>
#include <linux/kernel.h>
#include <linux/notifier.h>
#include <linux/signal.h>
-#include <linux/sched.h>
+#include <linux/sched/signal.h>
#include <linux/smp.h>
#include <linux/init.h>
#include <linux/uaccess.h>
.unmap_page = xen_swiotlb_unmap_page,
.dma_supported = xen_swiotlb_dma_supported,
.set_dma_mask = xen_swiotlb_set_dma_mask,
+ .mmap = xen_swiotlb_dma_mmap,
+ .get_sgtable = xen_swiotlb_get_sgtable,
};
int __init xen_mm_init(void)
If unsure, say Y.
+config QCOM_QDF2400_ERRATUM_0065
+ bool "QDF2400 E0065: Incorrect GITS_TYPER.ITT_Entry_size"
+ default y
+ help
+ On Qualcomm Datacenter Technologies QDF2400 SoC, ITS hardware reports
+ ITE size incorrectly. The GITS_TYPER.ITT_Entry_size field should have
+ been indicated as 16Bytes (0xf), not 8Bytes (0x7).
+
+ If unsure, say Y.
+
endmenu
dtb-$(CONFIG_ARCH_MESON) += meson-gxbb-vega-s95-pro.dtb
dtb-$(CONFIG_ARCH_MESON) += meson-gxbb-vega-s95-meta.dtb
dtb-$(CONFIG_ARCH_MESON) += meson-gxbb-vega-s95-telos.dtb
+dtb-$(CONFIG_ARCH_MESON) += meson-gxbb-wetek-hub.dtb
+dtb-$(CONFIG_ARCH_MESON) += meson-gxbb-wetek-play2.dtb
dtb-$(CONFIG_ARCH_MESON) += meson-gxl-s905x-p212.dtb
dtb-$(CONFIG_ARCH_MESON) += meson-gxl-s905d-p230.dtb
dtb-$(CONFIG_ARCH_MESON) += meson-gxl-s905d-p231.dtb
-dtb-$(CONFIG_ARCH_MESON) += meson-gxl-nexbox-a95x.dtb
-dtb-$(CONFIG_ARCH_MESON) += meson-gxm-s912-q200.dtb
-dtb-$(CONFIG_ARCH_MESON) += meson-gxm-s912-q201.dtb
+dtb-$(CONFIG_ARCH_MESON) += meson-gxl-s905x-nexbox-a95x.dtb
+dtb-$(CONFIG_ARCH_MESON) += meson-gxm-q200.dtb
+dtb-$(CONFIG_ARCH_MESON) += meson-gxm-q201.dtb
dtb-$(CONFIG_ARCH_MESON) += meson-gxm-nexbox-a1.dtb
always := $(dtb-y)
reg = <0x0 0x0>;
enable-method = "psci";
next-level-cache = <&l2>;
+ clocks = <&scpi_dvfs 0>;
};
cpu1: cpu@1 {
reg = <0x0 0x1>;
enable-method = "psci";
next-level-cache = <&l2>;
+ clocks = <&scpi_dvfs 0>;
};
cpu2: cpu@2 {
reg = <0x0 0x2>;
enable-method = "psci";
next-level-cache = <&l2>;
+ clocks = <&scpi_dvfs 0>;
};
cpu3: cpu@3 {
reg = <0x0 0x3>;
enable-method = "psci";
next-level-cache = <&l2>;
+ clocks = <&scpi_dvfs 0>;
};
l2: l2-cache0 {
};
};
+ scpi {
+ compatible = "amlogic,meson-gxbb-scpi", "arm,scpi-pre-1.0";
+ mboxes = <&mailbox 1 &mailbox 2>;
+ shmem = <&cpu_scp_lpri &cpu_scp_hpri>;
+
+ scpi_clocks: clocks {
+ compatible = "arm,scpi-clocks";
+
+ scpi_dvfs: scpi_clocks@0 {
+ compatible = "arm,scpi-dvfs-clocks";
+ #clock-cells = <1>;
+ clock-indices = <0>;
+ clock-output-names = "vcpu";
+ };
+ };
+
+ scpi_sensors: sensors {
+ compatible = "arm,scpi-sensors";
+ #thermal-sensor-cells = <1>;
+ };
+ };
+
soc {
compatible = "simple-bus";
#address-cells = <2>;
status = "disabled";
};
+ saradc: adc@8680 {
+ compatible = "amlogic,meson-saradc";
+ reg = <0x0 0x8680 0x0 0x34>;
+ #io-channel-cells = <1>;
+ interrupts = <GIC_SPI 73 IRQ_TYPE_EDGE_RISING>;
+ status = "disabled";
+ };
+
pwm_ef: pwm@86c0 {
compatible = "amlogic,meson-gx-pwm", "amlogic,meson-gxbb-pwm";
reg = <0x0 0x086c0 0x0 0x10>;
#address-cells = <0>;
};
+ sram: sram@c8000000 {
+ compatible = "amlogic,meson-gxbb-sram", "mmio-sram";
+ reg = <0x0 0xc8000000 0x0 0x14000>;
+
+ #address-cells = <1>;
+ #size-cells = <1>;
+ ranges = <0 0x0 0xc8000000 0x14000>;
+
+ cpu_scp_lpri: scp-shmem@0 {
+ compatible = "amlogic,meson-gxbb-scp-shmem";
+ reg = <0x13000 0x400>;
+ };
+
+ cpu_scp_hpri: scp-shmem@200 {
+ compatible = "amlogic,meson-gxbb-scp-shmem";
+ reg = <0x13400 0x400>;
+ };
+ };
+
aobus: aobus@c8100000 {
compatible = "simple-bus";
reg = <0x0 0xc8100000 0x0 0x100000>;
status = "disabled";
};
+ uart_AO_B: serial@4e0 {
+ compatible = "amlogic,meson-uart";
+ reg = <0x0 0x004e0 0x0 0x14>;
+ interrupts = <GIC_SPI 197 IRQ_TYPE_EDGE_RISING>;
+ clocks = <&xtal>;
+ status = "disabled";
+ };
+
+ pwm_AO_ab: pwm@550 {
+ compatible = "amlogic,meson-gx-pwm", "amlogic,meson-gxbb-pwm";
+ reg = <0x0 0x00550 0x0 0x10>;
+ #pwm-cells = <3>;
+ status = "disabled";
+ };
+
ir: ir@580 {
compatible = "amlogic,meson-gxbb-ir";
reg = <0x0 0x00580 0x0 0x40>;
/dts-v1/;
#include "meson-gxbb-p20x.dtsi"
+#include <dt-bindings/input/input.h>
/ {
compatible = "amlogic,p200", "amlogic,meson-gxbb";
model = "Amlogic Meson GXBB P200 Development Board";
+
+ avdd18_usb_adc: regulator-avdd18_usb_adc {
+ compatible = "regulator-fixed";
+ regulator-name = "AVDD18_USB_ADC";
+ regulator-min-microvolt = <1800000>;
+ regulator-max-microvolt = <1800000>;
+ };
+
+ adc_keys {
+ compatible = "adc-keys";
+ io-channels = <&saradc 0>;
+ io-channel-names = "buttons";
+ keyup-threshold-microvolt = <1800000>;
+
+ button-home {
+ label = "Home";
+ linux,code = <KEY_HOME>;
+ press-threshold-microvolt = <900000>; /* 50% */
+ };
+
+ button-esc {
+ label = "Esc";
+ linux,code = <KEY_ESC>;
+ press-threshold-microvolt = <684000>; /* 38% */
+ };
+
+ button-up {
+ label = "Volume Up";
+ linux,code = <KEY_VOLUMEUP>;
+ press-threshold-microvolt = <468000>; /* 26% */
+ };
+
+ button-down {
+ label = "Volume Down";
+ linux,code = <KEY_VOLUMEDOWN>;
+ press-threshold-microvolt = <252000>; /* 14% */
+ };
+
+ button-menu {
+ label = "Menu";
+ linux,code = <KEY_MENU>;
+ press-threshold-microvolt = <0>; /* 0% */
+ };
+ };
};
&i2c_B {
pinctrl-0 = <&i2c_b_pins>;
pinctrl-names = "default";
};
+
+&saradc {
+ status = "okay";
+ vref-supply = <&avdd18_usb_adc>;
+};
stdout-path = "serial0:115200n8";
};
+ leds {
+ compatible = "gpio-leds";
+
+ blue {
+ label = "vega-s95:blue:on";
+ gpios = <&gpio_ao GPIOAO_13 GPIO_ACTIVE_HIGH>;
+ default-state = "on";
+ panic-indicator;
+ };
+ };
+
usb_vbus: regulator-usb0-vbus {
compatible = "regulator-fixed";
--- /dev/null
+/*
+ * Copyright (c) 2016 BayLibre, Inc.
+ * Author: Neil Armstrong <narmstrong@baylibre.com>
+ *
+ * This file is dual-licensed: you can use it either under the terms
+ * of the GPL or the X11 license, at your option. Note that this dual
+ * licensing only applies to this file, and not this project as a
+ * whole.
+ *
+ * a) This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * Or, alternatively,
+ *
+ * b) Permission is hereby granted, free of charge, to any person
+ * obtaining a copy of this software and associated documentation
+ * files (the "Software"), to deal in the Software without
+ * restriction, including without limitation the rights to use,
+ * copy, modify, merge, publish, distribute, sublicense, and/or
+ * sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following
+ * conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+/dts-v1/;
+
+#include "meson-gxbb-p20x.dtsi"
+
+/ {
+ compatible = "wetek,hub", "amlogic,meson-gxbb";
+ model = "WeTek Hub";
+
+ leds {
+ compatible = "gpio-leds";
+
+ system {
+ label = "wetek-play:system-status";
+ gpios = <&gpio_ao GPIOAO_13 GPIO_ACTIVE_HIGH>;
+ default-state = "on";
+ panic-indicator;
+ };
+ };
+
+ cvbs-connector {
+ status = "disabled";
+ };
+};
--- /dev/null
+/*
+ * Copyright (c) 2016 BayLibre, Inc.
+ * Author: Neil Armstrong <narmstrong@baylibre.com>
+ *
+ * This file is dual-licensed: you can use it either under the terms
+ * of the GPL or the X11 license, at your option. Note that this dual
+ * licensing only applies to this file, and not this project as a
+ * whole.
+ *
+ * a) This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * Or, alternatively,
+ *
+ * b) Permission is hereby granted, free of charge, to any person
+ * obtaining a copy of this software and associated documentation
+ * files (the "Software"), to deal in the Software without
+ * restriction, including without limitation the rights to use,
+ * copy, modify, merge, publish, distribute, sublicense, and/or
+ * sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following
+ * conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+/dts-v1/;
+
+#include "meson-gxbb-p20x.dtsi"
+#include <dt-bindings/input/input.h>
+
+/ {
+ compatible = "wetek,play2", "amlogic,meson-gxbb";
+ model = "WeTek Play 2";
+
+ leds {
+ compatible = "gpio-leds";
+
+ system {
+ label = "wetek-play:system-status";
+ gpios = <&gpio_ao GPIOAO_13 GPIO_ACTIVE_HIGH>;
+ default-state = "on";
+ panic-indicator;
+ };
+
+ wifi {
+ label = "wetek-play:wifi-status";
+ gpios = <&gpio GPIODV_26 GPIO_ACTIVE_HIGH>;
+ default-state = "off";
+ };
+
+ ethernet {
+ label = "wetek-play:ethernet-status";
+ gpios = <&gpio GPIODV_27 GPIO_ACTIVE_HIGH>;
+ default-state = "off";
+ };
+ };
+
+ gpio-keys-polled {
+ compatible = "gpio-keys-polled";
+ #address-cells = <1>;
+ #size-cells = <0>;
+ poll-interval = <100>;
+
+ button@0 {
+ label = "reset";
+ linux,code = <KEY_RESTART>;
+ gpios = <&gpio_ao GPIOAO_3 GPIO_ACTIVE_LOW>;
+ };
+ };
+};
+
+&i2c_A {
+ status = "okay";
+ pinctrl-0 = <&i2c_a_pins>;
+ pinctrl-names = "default";
+};
/ {
compatible = "amlogic,meson-gxbb";
- scpi {
- compatible = "amlogic,meson-gxbb-scpi", "arm,scpi-pre-1.0";
- mboxes = <&mailbox 1 &mailbox 2>;
- shmem = <&cpu_scp_lpri &cpu_scp_hpri>;
-
- scpi_clocks: clocks {
- compatible = "arm,scpi-clocks";
-
- scpi_dvfs: scpi_clocks@0 {
- compatible = "arm,scpi-dvfs-clocks";
- #clock-cells = <1>;
- clock-indices = <0>;
- clock-output-names = "vcpu";
- };
- };
-
- scpi_sensors: sensors {
- compatible = "arm,scpi-sensors";
- #thermal-sensor-cells = <1>;
- };
- };
-
soc {
usb0_phy: phy@c0000000 {
compatible = "amlogic,meson-gxbb-usb2-phy";
status = "disabled";
};
- sram: sram@c8000000 {
- compatible = "amlogic,meson-gxbb-sram", "mmio-sram";
- reg = <0x0 0xc8000000 0x0 0x14000>;
-
- #address-cells = <1>;
- #size-cells = <1>;
- ranges = <0 0x0 0xc8000000 0x14000>;
-
- cpu_scp_lpri: scp-shmem@0 {
- compatible = "amlogic,meson-gxbb-scp-shmem";
- reg = <0x13000 0x400>;
- };
-
- cpu_scp_hpri: scp-shmem@200 {
- compatible = "amlogic,meson-gxbb-scp-shmem";
- reg = <0x13400 0x400>;
- };
- };
-
usb0: usb@c9000000 {
compatible = "amlogic,meson-gxbb-usb", "snps,dwc2";
reg = <0x0 0xc9000000 0x0 0x40000>;
};
};
-&cpu0 {
- clocks = <&scpi_dvfs 0>;
-};
-
-&cpu1 {
- clocks = <&scpi_dvfs 0>;
-};
-
-&cpu2 {
- clocks = <&scpi_dvfs 0>;
-};
-
-&cpu3 {
- clocks = <&scpi_dvfs 0>;
-};
-
&cbus {
spifc: spi@8c80 {
compatible = "amlogic,meson-gxbb-spifc";
};
};
+ uart_ao_a_cts_rts_pins: uart_ao_a_cts_rts {
+ mux {
+ groups = "uart_cts_ao_a",
+ "uart_rts_ao_a";
+ function = "uart_ao";
+ };
+ };
+
+ uart_ao_b_pins: uart_ao_b {
+ mux {
+ groups = "uart_tx_ao_b", "uart_rx_ao_b";
+ function = "uart_ao_b";
+ };
+ };
+
+ uart_ao_b_cts_rts_pins: uart_ao_b_cts_rts {
+ mux {
+ groups = "uart_cts_ao_b",
+ "uart_rts_ao_b";
+ function = "uart_ao_b";
+ };
+ };
+
remote_input_ao_pins: remote_input_ao {
mux {
groups = "remote_input_ao";
};
};
+ uart_a_cts_rts_pins: uart_a_cts_rts {
+ mux {
+ groups = "uart_cts_a",
+ "uart_rts_a";
+ function = "uart_a";
+ };
+ };
+
uart_b_pins: uart_b {
mux {
groups = "uart_tx_b",
};
};
+ uart_b_cts_rts_pins: uart_b_cts_rts {
+ mux {
+ groups = "uart_cts_b",
+ "uart_rts_b";
+ function = "uart_b";
+ };
+ };
+
uart_c_pins: uart_c {
mux {
groups = "uart_tx_c",
};
};
+ uart_c_cts_rts_pins: uart_c_cts_rts {
+ mux {
+ groups = "uart_cts_c",
+ "uart_rts_c";
+ function = "uart_c";
+ };
+ };
+
i2c_a_pins: i2c_a {
mux {
groups = "i2c_sck_a",
function = "pwm_f_y";
};
};
+
+ hdmi_hpd_pins: hdmi_hpd {
+ mux {
+ groups = "hdmi_hpd";
+ function = "hdmi_hpd";
+ };
+ };
+
+ hdmi_i2c_pins: hdmi_i2c {
+ mux {
+ groups = "hdmi_sda", "hdmi_scl";
+ function = "hdmi_i2c";
+ };
+ };
};
};
clocks = <&clkc CLKID_I2C>;
};
+&saradc {
+ compatible = "amlogic,meson-gxbb-saradc", "amlogic,meson-saradc";
+ clocks = <&xtal>,
+ <&clkc CLKID_SAR_ADC>,
+ <&clkc CLKID_SANA>,
+ <&clkc CLKID_SAR_ADC_CLK>,
+ <&clkc CLKID_SAR_ADC_SEL>;
+ clock-names = "clkin", "core", "sana", "adc_clk", "adc_sel";
+};
+
&sd_emmc_a {
clocks = <&clkc CLKID_SD_EMMC_A>,
<&xtal>,
+++ /dev/null
-/*
- * Copyright (c) 2016 Andreas Färber
- * Copyright (c) 2016 BayLibre, Inc.
- * Author: Neil Armstrong <narmstrong@kernel.org>
- *
- * This file is dual-licensed: you can use it either under the terms
- * of the GPL or the X11 license, at your option. Note that this dual
- * licensing only applies to this file, and not this project as a
- * whole.
- *
- * a) This library is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License as
- * published by the Free Software Foundation; either version 2 of the
- * License, or (at your option) any later version.
- *
- * This library is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU General Public License for more details.
- *
- * Or, alternatively,
- *
- * b) Permission is hereby granted, free of charge, to any person
- * obtaining a copy of this software and associated documentation
- * files (the "Software"), to deal in the Software without
- * restriction, including without limitation the rights to use,
- * copy, modify, merge, publish, distribute, sublicense, and/or
- * sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following
- * conditions:
- *
- * The above copyright notice and this permission notice shall be
- * included in all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
- * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
- * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
- * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
- * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
- * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
- * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
- * OTHER DEALINGS IN THE SOFTWARE.
- */
-
-/dts-v1/;
-
-#include "meson-gxl-s905x.dtsi"
-
-/ {
- compatible = "nexbox,a95x", "amlogic,s905x", "amlogic,meson-gxl";
- model = "NEXBOX A95X (S905X)";
-
- aliases {
- serial0 = &uart_AO;
- };
-
- chosen {
- stdout-path = "serial0:115200n8";
- };
-
- memory@0 {
- device_type = "memory";
- reg = <0x0 0x0 0x0 0x80000000>;
- };
-
- vddio_card: gpio-regulator {
- compatible = "regulator-gpio";
-
- regulator-name = "VDDIO_CARD";
- regulator-min-microvolt = <1800000>;
- regulator-max-microvolt = <3300000>;
-
- gpios = <&gpio_ao GPIOAO_5 GPIO_ACTIVE_HIGH>;
- gpios-states = <1>;
-
- /* Based on P200 schematics, signal CARD_1.8V/3.3V_CTR */
- states = <1800000 0
- 3300000 1>;
- };
-
- vddio_boot: regulator-vddio_boot {
- compatible = "regulator-fixed";
- regulator-name = "VDDIO_BOOT";
- regulator-min-microvolt = <1800000>;
- regulator-max-microvolt = <1800000>;
- };
-
- vddao_3v3: regulator-vddao_3v3 {
- compatible = "regulator-fixed";
- regulator-name = "VDDAO_3V3";
- regulator-min-microvolt = <3300000>;
- regulator-max-microvolt = <3300000>;
- };
-
- vcc_3v3: regulator-vcc_3v3 {
- compatible = "regulator-fixed";
- regulator-name = "VCC_3V3";
- regulator-min-microvolt = <3300000>;
- regulator-max-microvolt = <3300000>;
- };
-
- emmc_pwrseq: emmc-pwrseq {
- compatible = "mmc-pwrseq-emmc";
- reset-gpios = <&gpio BOOT_9 GPIO_ACTIVE_LOW>;
- };
-
- wifi32k: wifi32k {
- compatible = "pwm-clock";
- #clock-cells = <0>;
- clock-frequency = <32768>;
- pwms = <&pwm_ef 0 30518 0>; /* PWM_E at 32.768KHz */
- };
-
- sdio_pwrseq: sdio-pwrseq {
- compatible = "mmc-pwrseq-simple";
- reset-gpios = <&gpio GPIOX_6 GPIO_ACTIVE_LOW>;
- clocks = <&wifi32k>;
- clock-names = "ext_clock";
- };
-
- cvbs-connector {
- compatible = "composite-video-connector";
-
- port {
- cvbs_connector_in: endpoint {
- remote-endpoint = <&cvbs_vdac_out>;
- };
- };
- };
-};
-
-&uart_AO {
- status = "okay";
- pinctrl-0 = <&uart_ao_a_pins>;
- pinctrl-names = "default";
-};
-
-ðmac {
- status = "okay";
- phy-mode = "rmii";
- phy-handle = <&internal_phy>;
-};
-
-&ir {
- status = "okay";
- pinctrl-0 = <&remote_input_ao_pins>;
- pinctrl-names = "default";
-};
-
-/* Wireless SDIO Module */
-&sd_emmc_a {
- status = "okay";
- pinctrl-0 = <&sdio_pins>;
- pinctrl-names = "default";
- #address-cells = <1>;
- #size-cells = <0>;
-
- bus-width = <4>;
- cap-sd-highspeed;
- max-frequency = <100000000>;
-
- non-removable;
- disable-wp;
-
- mmc-pwrseq = <&sdio_pwrseq>;
-
- vmmc-supply = <&vddao_3v3>;
- vqmmc-supply = <&vddio_boot>;
-};
-
-/* SD card */
-&sd_emmc_b {
- status = "okay";
- pinctrl-0 = <&sdcard_pins>;
- pinctrl-names = "default";
-
- bus-width = <4>;
- cap-sd-highspeed;
- max-frequency = <100000000>;
- disable-wp;
-
- cd-gpios = <&gpio CARD_6 GPIO_ACTIVE_HIGH>;
- cd-inverted;
-
- vmmc-supply = <&vddao_3v3>;
- vqmmc-supply = <&vddio_card>;
-};
-
-/* eMMC */
-&sd_emmc_c {
- status = "okay";
- pinctrl-0 = <&emmc_pins>;
- pinctrl-names = "default";
-
- bus-width = <8>;
- cap-sd-highspeed;
- cap-mmc-highspeed;
- max-frequency = <200000000>;
- non-removable;
- disable-wp;
- mmc-ddr-1_8v;
- mmc-hs200-1_8v;
-
- mmc-pwrseq = <&emmc_pwrseq>;
- vmmc-supply = <&vcc_3v3>;
- vqmmc-supply = <&vddio_boot>;
-};
-
-&pwm_ef {
- status = "okay";
- pinctrl-0 = <&pwm_e_pins>;
- pinctrl-names = "default";
- clocks = <&clkc CLKID_FCLK_DIV4>;
- clock-names = "clkin0";
-};
-
-&cvbs_vdac_port {
- cvbs_vdac_out: endpoint {
- remote-endpoint = <&cvbs_connector_in>;
- };
-};
--- /dev/null
+/*
+ * Copyright (c) 2016 Andreas Färber
+ * Copyright (c) 2016 BayLibre, Inc.
+ * Author: Neil Armstrong <narmstrong@kernel.org>
+ *
+ * This file is dual-licensed: you can use it either under the terms
+ * of the GPL or the X11 license, at your option. Note that this dual
+ * licensing only applies to this file, and not this project as a
+ * whole.
+ *
+ * a) This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * Or, alternatively,
+ *
+ * b) Permission is hereby granted, free of charge, to any person
+ * obtaining a copy of this software and associated documentation
+ * files (the "Software"), to deal in the Software without
+ * restriction, including without limitation the rights to use,
+ * copy, modify, merge, publish, distribute, sublicense, and/or
+ * sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following
+ * conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+/dts-v1/;
+
+#include "meson-gxl-s905x.dtsi"
+
+/ {
+ compatible = "nexbox,a95x", "amlogic,s905x", "amlogic,meson-gxl";
+ model = "NEXBOX A95X (S905X)";
+
+ aliases {
+ serial0 = &uart_AO;
+ };
+
+ chosen {
+ stdout-path = "serial0:115200n8";
+ };
+
+ memory@0 {
+ device_type = "memory";
+ reg = <0x0 0x0 0x0 0x80000000>;
+ };
+
+ vddio_card: gpio-regulator {
+ compatible = "regulator-gpio";
+
+ regulator-name = "VDDIO_CARD";
+ regulator-min-microvolt = <1800000>;
+ regulator-max-microvolt = <3300000>;
+
+ gpios = <&gpio_ao GPIOAO_5 GPIO_ACTIVE_HIGH>;
+ gpios-states = <1>;
+
+ /* Based on P200 schematics, signal CARD_1.8V/3.3V_CTR */
+ states = <1800000 0
+ 3300000 1>;
+ };
+
+ vddio_boot: regulator-vddio_boot {
+ compatible = "regulator-fixed";
+ regulator-name = "VDDIO_BOOT";
+ regulator-min-microvolt = <1800000>;
+ regulator-max-microvolt = <1800000>;
+ };
+
+ vddao_3v3: regulator-vddao_3v3 {
+ compatible = "regulator-fixed";
+ regulator-name = "VDDAO_3V3";
+ regulator-min-microvolt = <3300000>;
+ regulator-max-microvolt = <3300000>;
+ };
+
+ vcc_3v3: regulator-vcc_3v3 {
+ compatible = "regulator-fixed";
+ regulator-name = "VCC_3V3";
+ regulator-min-microvolt = <3300000>;
+ regulator-max-microvolt = <3300000>;
+ };
+
+ emmc_pwrseq: emmc-pwrseq {
+ compatible = "mmc-pwrseq-emmc";
+ reset-gpios = <&gpio BOOT_9 GPIO_ACTIVE_LOW>;
+ };
+
+ wifi32k: wifi32k {
+ compatible = "pwm-clock";
+ #clock-cells = <0>;
+ clock-frequency = <32768>;
+ pwms = <&pwm_ef 0 30518 0>; /* PWM_E at 32.768KHz */
+ };
+
+ sdio_pwrseq: sdio-pwrseq {
+ compatible = "mmc-pwrseq-simple";
+ reset-gpios = <&gpio GPIOX_6 GPIO_ACTIVE_LOW>;
+ clocks = <&wifi32k>;
+ clock-names = "ext_clock";
+ };
+
+ cvbs-connector {
+ compatible = "composite-video-connector";
+
+ port {
+ cvbs_connector_in: endpoint {
+ remote-endpoint = <&cvbs_vdac_out>;
+ };
+ };
+ };
+};
+
+&uart_AO {
+ status = "okay";
+ pinctrl-0 = <&uart_ao_a_pins>;
+ pinctrl-names = "default";
+};
+
+ðmac {
+ status = "okay";
+ phy-mode = "rmii";
+ phy-handle = <&internal_phy>;
+};
+
+&ir {
+ status = "okay";
+ pinctrl-0 = <&remote_input_ao_pins>;
+ pinctrl-names = "default";
+};
+
+/* Wireless SDIO Module */
+&sd_emmc_a {
+ status = "okay";
+ pinctrl-0 = <&sdio_pins>;
+ pinctrl-names = "default";
+ #address-cells = <1>;
+ #size-cells = <0>;
+
+ bus-width = <4>;
+ cap-sd-highspeed;
+ max-frequency = <100000000>;
+
+ non-removable;
+ disable-wp;
+
+ mmc-pwrseq = <&sdio_pwrseq>;
+
+ vmmc-supply = <&vddao_3v3>;
+ vqmmc-supply = <&vddio_boot>;
+};
+
+/* SD card */
+&sd_emmc_b {
+ status = "okay";
+ pinctrl-0 = <&sdcard_pins>;
+ pinctrl-names = "default";
+
+ bus-width = <4>;
+ cap-sd-highspeed;
+ max-frequency = <100000000>;
+ disable-wp;
+
+ cd-gpios = <&gpio CARD_6 GPIO_ACTIVE_HIGH>;
+ cd-inverted;
+
+ vmmc-supply = <&vddao_3v3>;
+ vqmmc-supply = <&vddio_card>;
+};
+
+/* eMMC */
+&sd_emmc_c {
+ status = "okay";
+ pinctrl-0 = <&emmc_pins>;
+ pinctrl-names = "default";
+
+ bus-width = <8>;
+ cap-sd-highspeed;
+ cap-mmc-highspeed;
+ max-frequency = <200000000>;
+ non-removable;
+ disable-wp;
+ mmc-ddr-1_8v;
+ mmc-hs200-1_8v;
+
+ mmc-pwrseq = <&emmc_pwrseq>;
+ vmmc-supply = <&vcc_3v3>;
+ vqmmc-supply = <&vddio_boot>;
+};
+
+&pwm_ef {
+ status = "okay";
+ pinctrl-0 = <&pwm_e_pins>;
+ pinctrl-names = "default";
+ clocks = <&clkc CLKID_FCLK_DIV4>;
+ clock-names = "clkin0";
+};
+
+&cvbs_vdac_port {
+ cvbs_vdac_out: endpoint {
+ remote-endpoint = <&cvbs_connector_in>;
+ };
+};
};
};
+ uart_ao_a_cts_rts_pins: uart_ao_a_cts_rts {
+ mux {
+ groups = "uart_cts_ao_a",
+ "uart_rts_ao_a";
+ function = "uart_ao";
+ };
+ };
+
+ uart_ao_b_pins: uart_ao_b {
+ mux {
+ groups = "uart_tx_ao_b", "uart_rx_ao_b";
+ function = "uart_ao_b";
+ };
+ };
+
+ uart_ao_b_cts_rts_pins: uart_ao_b_cts_rts {
+ mux {
+ groups = "uart_cts_ao_b",
+ "uart_rts_ao_b";
+ function = "uart_ao_b";
+ };
+ };
+
remote_input_ao_pins: remote_input_ao {
mux {
groups = "remote_input_ao";
function = "remote_input_ao";
};
};
+
+ pwm_ao_b_pins: pwm_ao_b {
+ mux {
+ groups = "pwm_ao_b";
+ function = "pwm_ao_b";
+ };
+ };
};
};
};
};
+ uart_a_cts_rts_pins: uart_a_cts_rts {
+ mux {
+ groups = "uart_cts_a",
+ "uart_rts_a";
+ function = "uart_a";
+ };
+ };
+
uart_b_pins: uart_b {
mux {
groups = "uart_tx_b",
};
};
+ uart_b_cts_rts_pins: uart_b_cts_rts {
+ mux {
+ groups = "uart_cts_b",
+ "uart_rts_b";
+ function = "uart_b";
+ };
+ };
+
uart_c_pins: uart_c {
mux {
groups = "uart_tx_c",
};
};
+ uart_c_cts_rts_pins: uart_c_cts_rts {
+ mux {
+ groups = "uart_cts_c",
+ "uart_rts_c";
+ function = "uart_c";
+ };
+ };
+
i2c_a_pins: i2c_a {
mux {
groups = "i2c_sck_a",
function = "pwm_e";
};
};
+
+ hdmi_hpd_pins: hdmi_hpd {
+ mux {
+ groups = "hdmi_hpd";
+ function = "hdmi_hpd";
+ };
+ };
+
+ hdmi_i2c_pins: hdmi_i2c {
+ mux {
+ groups = "hdmi_sda", "hdmi_scl";
+ function = "hdmi_i2c";
+ };
+ };
};
eth-phy-mux {
clocks = <&clkc CLKID_I2C>;
};
+&saradc {
+ compatible = "amlogic,meson-gxl-saradc", "amlogic,meson-saradc";
+ clocks = <&xtal>,
+ <&clkc CLKID_SAR_ADC>,
+ <&clkc CLKID_SANA>,
+ <&clkc CLKID_SAR_ADC_CLK>,
+ <&clkc CLKID_SAR_ADC_SEL>;
+ clock-names = "clkin", "core", "sana", "adc_clk", "adc_sel";
+};
+
&sd_emmc_a {
clocks = <&clkc CLKID_SD_EMMC_A>,
<&xtal>,
--- /dev/null
+/*
+ * Copyright (c) 2016 Endless Computers, Inc.
+ * Author: Carlo Caione <carlo@endlessm.com>
+ *
+ * This file is dual-licensed: you can use it either under the terms
+ * of the GPL or the X11 license, at your option. Note that this dual
+ * licensing only applies to this file, and not this project as a
+ * whole.
+ *
+ * a) This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * Or, alternatively,
+ *
+ * b) Permission is hereby granted, free of charge, to any person
+ * obtaining a copy of this software and associated documentation
+ * files (the "Software"), to deal in the Software without
+ * restriction, including without limitation the rights to use,
+ * copy, modify, merge, publish, distribute, sublicense, and/or
+ * sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following
+ * conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+/dts-v1/;
+
+#include "meson-gxm.dtsi"
+#include "meson-gx-p23x-q20x.dtsi"
+
+/ {
+ compatible = "amlogic,q200", "amlogic,s912", "amlogic,meson-gxm";
+ model = "Amlogic Meson GXM (S912) Q200 Development Board";
+};
+
+/* Q200 has exclusive choice between internal or external PHY */
+ðmac {
+ pinctrl-0 = <ð_pins>;
+ pinctrl-names = "default";
+
+ /* Select external PHY by default */
+ phy-handle = <&external_phy>;
+
+ /* External PHY reset is shared with internal PHY Led signals */
+ snps,reset-gpio = <&gpio GPIOZ_14 0>;
+ snps,reset-delays-us = <0 10000 1000000>;
+ snps,reset-active-low;
+
+ /* External PHY is in RGMII */
+ phy-mode = "rgmii";
+};
+
+&external_mdio {
+ external_phy: ethernet-phy@0 {
+ compatible = "ethernet-phy-id001c.c916", "ethernet-phy-ieee802.3-c22";
+ reg = <0>;
+ max-speed = <1000>;
+ };
+};
--- /dev/null
+/*
+ * Copyright (c) 2016 Endless Computers, Inc.
+ * Author: Carlo Caione <carlo@endlessm.com>
+ *
+ * This file is dual-licensed: you can use it either under the terms
+ * of the GPL or the X11 license, at your option. Note that this dual
+ * licensing only applies to this file, and not this project as a
+ * whole.
+ *
+ * a) This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * Or, alternatively,
+ *
+ * b) Permission is hereby granted, free of charge, to any person
+ * obtaining a copy of this software and associated documentation
+ * files (the "Software"), to deal in the Software without
+ * restriction, including without limitation the rights to use,
+ * copy, modify, merge, publish, distribute, sublicense, and/or
+ * sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following
+ * conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+/dts-v1/;
+
+#include "meson-gxm.dtsi"
+#include "meson-gx-p23x-q20x.dtsi"
+
+/ {
+ compatible = "amlogic,q201", "amlogic,s912", "amlogic,meson-gxm";
+ model = "Amlogic Meson GXM (S912) Q201 Development Board";
+};
+
+/* Q201 has only internal PHY port */
+ðmac {
+ phy-mode = "rmii";
+ phy-handle = <&internal_phy>;
+};
+++ /dev/null
-/*
- * Copyright (c) 2016 Endless Computers, Inc.
- * Author: Carlo Caione <carlo@endlessm.com>
- *
- * This file is dual-licensed: you can use it either under the terms
- * of the GPL or the X11 license, at your option. Note that this dual
- * licensing only applies to this file, and not this project as a
- * whole.
- *
- * a) This library is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License as
- * published by the Free Software Foundation; either version 2 of the
- * License, or (at your option) any later version.
- *
- * This library is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU General Public License for more details.
- *
- * Or, alternatively,
- *
- * b) Permission is hereby granted, free of charge, to any person
- * obtaining a copy of this software and associated documentation
- * files (the "Software"), to deal in the Software without
- * restriction, including without limitation the rights to use,
- * copy, modify, merge, publish, distribute, sublicense, and/or
- * sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following
- * conditions:
- *
- * The above copyright notice and this permission notice shall be
- * included in all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
- * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
- * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
- * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
- * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
- * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
- * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
- * OTHER DEALINGS IN THE SOFTWARE.
- */
-
-/dts-v1/;
-
-#include "meson-gxm.dtsi"
-#include "meson-gx-p23x-q20x.dtsi"
-
-/ {
- compatible = "amlogic,q200", "amlogic,s912", "amlogic,meson-gxm";
- model = "Amlogic Meson GXM (S912) Q200 Development Board";
-};
-
-/* Q200 has exclusive choice between internal or external PHY */
-ðmac {
- pinctrl-0 = <ð_pins>;
- pinctrl-names = "default";
-
- /* Select external PHY by default */
- phy-handle = <&external_phy>;
-
- /* External PHY reset is shared with internal PHY Led signals */
- snps,reset-gpio = <&gpio GPIOZ_14 0>;
- snps,reset-delays-us = <0 10000 1000000>;
- snps,reset-active-low;
-
- /* External PHY is in RGMII */
- phy-mode = "rgmii";
-};
-
-&external_mdio {
- external_phy: ethernet-phy@0 {
- compatible = "ethernet-phy-id001c.c916", "ethernet-phy-ieee802.3-c22";
- reg = <0>;
- max-speed = <1000>;
- };
-};
+++ /dev/null
-/*
- * Copyright (c) 2016 Endless Computers, Inc.
- * Author: Carlo Caione <carlo@endlessm.com>
- *
- * This file is dual-licensed: you can use it either under the terms
- * of the GPL or the X11 license, at your option. Note that this dual
- * licensing only applies to this file, and not this project as a
- * whole.
- *
- * a) This library is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License as
- * published by the Free Software Foundation; either version 2 of the
- * License, or (at your option) any later version.
- *
- * This library is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU General Public License for more details.
- *
- * Or, alternatively,
- *
- * b) Permission is hereby granted, free of charge, to any person
- * obtaining a copy of this software and associated documentation
- * files (the "Software"), to deal in the Software without
- * restriction, including without limitation the rights to use,
- * copy, modify, merge, publish, distribute, sublicense, and/or
- * sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following
- * conditions:
- *
- * The above copyright notice and this permission notice shall be
- * included in all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
- * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
- * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
- * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
- * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
- * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
- * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
- * OTHER DEALINGS IN THE SOFTWARE.
- */
-
-/dts-v1/;
-
-#include "meson-gxm.dtsi"
-#include "meson-gx-p23x-q20x.dtsi"
-
-/ {
- compatible = "amlogic,q201", "amlogic,s912", "amlogic,meson-gxm";
- model = "Amlogic Meson GXM (S912) Q201 Development Board";
-};
-
-/* Q201 has only internal PHY port */
-ðmac {
- phy-mode = "rmii";
- phy-handle = <&internal_phy>;
-};
reg = <0x0 0x100>;
enable-method = "psci";
next-level-cache = <&l2>;
+ clocks = <&scpi_dvfs 1>;
};
cpu5: cpu@101 {
reg = <0x0 0x101>;
enable-method = "psci";
next-level-cache = <&l2>;
+ clocks = <&scpi_dvfs 1>;
};
cpu6: cpu@102 {
reg = <0x0 0x102>;
enable-method = "psci";
next-level-cache = <&l2>;
+ clocks = <&scpi_dvfs 1>;
};
cpu7: cpu@103 {
reg = <0x0 0x103>;
enable-method = "psci";
next-level-cache = <&l2>;
+ clocks = <&scpi_dvfs 1>;
};
};
};
+&saradc {
+ compatible = "amlogic,meson-gxm-saradc", "amlogic,meson-saradc";
+};
+
+&scpi_dvfs {
+ clock-indices = <0 1>;
+ clock-output-names = "vbig", "vlittle";
+};
+
&vpu {
compatible = "amlogic,meson-gxm-vpu", "amlogic,meson-gx-vpu";
};
+
};
};
- coresight-replicator {
- /*
- * Non-configurable replicators don't show up on the
- * AMBA bus. As such no need to add "arm,primecell".
- */
- compatible = "arm,coresight-replicator";
+ replicator@20120000 {
+ compatible = "qcom,coresight-replicator1x", "arm,primecell";
+ reg = <0 0x20120000 0 0x1000>;
+
+ clocks = <&soc_smc50mhz>;
+ clock-names = "apb_pclk";
+ power-domains = <&scpi_devpd 0>;
ports {
#address-cells = <1>;
assigned-clock-parents = <&cmu_top CLK_FOUT_AUD_PLL>;
};
-&cmu_disp {
- assigned-clocks = <&cmu_mif CLK_MOUT_SCLK_DECON_TV_ECLK_A>,
- <&cmu_mif CLK_DIV_SCLK_DECON_TV_ECLK>,
- <&cmu_disp CLK_MOUT_SCLK_DECON_TV_ECLK_USER>,
- <&cmu_disp CLK_MOUT_SCLK_DECON_TV_ECLK>;
- assigned-clock-parents = <&cmu_mif CLK_MOUT_BUS_PLL_DIV2>,
- <0>,
- <&cmu_mif CLK_SCLK_DECON_TV_ECLK_DISP>,
- <&cmu_disp CLK_MOUT_SCLK_DECON_TV_ECLK_USER>;
- assigned-clock-rates = <0>, <400000000>;
-};
-
&cmu_fsys {
assigned-clocks = <&cmu_top CLK_MOUT_SCLK_USBDRD30>,
<&cmu_top CLK_MOUT_SCLK_USBHOST30>,
compatible = "samsung,tm2", "samsung,exynos5433";
};
+&cmu_disp {
+ /*
+ * TM2 and TM2e differ only by DISP_PLL rate, but define all assigned
+ * clocks properties for DISP CMU for each board to keep them together
+ * for easier review and maintenance.
+ */
+ assigned-clocks = <&cmu_disp CLK_FOUT_DISP_PLL>,
+ <&cmu_mif CLK_DIV_SCLK_DECON_TV_ECLK>,
+ <&cmu_disp CLK_MOUT_ACLK_DISP_333_USER>,
+ <&cmu_disp CLK_MOUT_SCLK_DSIM0_USER>,
+ <&cmu_disp CLK_MOUT_SCLK_DSIM0>,
+ <&cmu_disp CLK_MOUT_SCLK_DECON_ECLK_USER>,
+ <&cmu_disp CLK_MOUT_SCLK_DECON_ECLK>,
+ <&cmu_disp CLK_MOUT_PHYCLK_MIPIDPHY0_RXCLKESC0_USER>,
+ <&cmu_disp CLK_MOUT_PHYCLK_MIPIDPHY0_BITCLKDIV8_USER>,
+ <&cmu_disp CLK_MOUT_DISP_PLL>,
+ <&cmu_mif CLK_MOUT_SCLK_DECON_TV_ECLK_A>,
+ <&cmu_disp CLK_MOUT_SCLK_DECON_TV_ECLK_USER>,
+ <&cmu_disp CLK_MOUT_SCLK_DECON_TV_ECLK>;
+ assigned-clock-parents = <0>, <0>,
+ <&cmu_mif CLK_ACLK_DISP_333>,
+ <&cmu_mif CLK_SCLK_DSIM0_DISP>,
+ <&cmu_disp CLK_MOUT_SCLK_DSIM0_USER>,
+ <&cmu_mif CLK_SCLK_DECON_ECLK_DISP>,
+ <&cmu_disp CLK_MOUT_SCLK_DECON_ECLK_USER>,
+ <&cmu_disp CLK_PHYCLK_MIPIDPHY0_RXCLKESC0_PHY>,
+ <&cmu_disp CLK_PHYCLK_MIPIDPHY0_BITCLKDIV8_PHY>,
+ <&cmu_disp CLK_FOUT_DISP_PLL>,
+ <&cmu_mif CLK_MOUT_BUS_PLL_DIV2>,
+ <&cmu_mif CLK_SCLK_DECON_TV_ECLK_DISP>,
+ <&cmu_disp CLK_MOUT_SCLK_DECON_TV_ECLK_USER>;
+ assigned-clock-rates = <250000000>, <400000000>;
+};
+
&hsi2c_9 {
status = "okay";
compatible = "samsung,tm2e", "samsung,exynos5433";
};
+&cmu_disp {
+ /*
+ * TM2 and TM2e differ only by DISP_PLL rate, but define all assigned
+ * clocks properties for DISP CMU for each board to keep them together
+ * for easier review and maintenance.
+ */
+ assigned-clocks = <&cmu_disp CLK_FOUT_DISP_PLL>,
+ <&cmu_mif CLK_DIV_SCLK_DECON_TV_ECLK>,
+ <&cmu_disp CLK_MOUT_ACLK_DISP_333_USER>,
+ <&cmu_disp CLK_MOUT_SCLK_DSIM0_USER>,
+ <&cmu_disp CLK_MOUT_SCLK_DSIM0>,
+ <&cmu_disp CLK_MOUT_SCLK_DECON_ECLK_USER>,
+ <&cmu_disp CLK_MOUT_SCLK_DECON_ECLK>,
+ <&cmu_disp CLK_MOUT_PHYCLK_MIPIDPHY0_RXCLKESC0_USER>,
+ <&cmu_disp CLK_MOUT_PHYCLK_MIPIDPHY0_BITCLKDIV8_USER>,
+ <&cmu_disp CLK_MOUT_DISP_PLL>,
+ <&cmu_mif CLK_MOUT_SCLK_DECON_TV_ECLK_A>,
+ <&cmu_disp CLK_MOUT_SCLK_DECON_TV_ECLK_USER>,
+ <&cmu_disp CLK_MOUT_SCLK_DECON_TV_ECLK>;
+ assigned-clock-parents = <0>, <0>,
+ <&cmu_mif CLK_ACLK_DISP_333>,
+ <&cmu_mif CLK_SCLK_DSIM0_DISP>,
+ <&cmu_disp CLK_MOUT_SCLK_DSIM0_USER>,
+ <&cmu_mif CLK_SCLK_DECON_ECLK_DISP>,
+ <&cmu_disp CLK_MOUT_SCLK_DECON_ECLK_USER>,
+ <&cmu_disp CLK_PHYCLK_MIPIDPHY0_RXCLKESC0_PHY>,
+ <&cmu_disp CLK_PHYCLK_MIPIDPHY0_BITCLKDIV8_PHY>,
+ <&cmu_disp CLK_FOUT_DISP_PLL>,
+ <&cmu_mif CLK_MOUT_BUS_PLL_DIV2>,
+ <&cmu_mif CLK_SCLK_DECON_TV_ECLK_DISP>,
+ <&cmu_disp CLK_MOUT_SCLK_DECON_TV_ECLK_USER>;
+ assigned-clock-rates = <278000000>, <400000000>;
+};
+
&ldo31_reg {
regulator-name = "TSP_VDD_1.8V_AP";
regulator-min-microvolt = <1800000>;
#include "exynos7.dtsi"
#include <dt-bindings/interrupt-controller/irq.h>
#include <dt-bindings/clock/samsung,s2mps11.h>
+#include <dt-bindings/gpio/gpio.h>
/ {
model = "Samsung Exynos7 Espresso board based on EXYNOS7";
device_type = "memory";
reg = <0x0 0x40000000 0x0 0xC0000000>;
};
+
+ usb30_vbus_reg: regulator-usb30 {
+ compatible = "regulator-fixed";
+ regulator-name = "VBUS_5V";
+ regulator-min-microvolt = <5000000>;
+ regulator-max-microvolt = <5000000>;
+ gpio = <&gph1 1 GPIO_ACTIVE_HIGH>;
+ pinctrl-names = "default";
+ pinctrl-0 = <&usb30_vbus_en>;
+ enable-active-high;
+ };
+
+ usb3drd_boost_5v: regulator-usb3drd-boost {
+ compatible = "regulator-fixed";
+ regulator-name = "VUSB_VBUS_5V";
+ regulator-min-microvolt = <5000000>;
+ regulator-max-microvolt = <5000000>;
+ gpio = <&gpf4 1 GPIO_ACTIVE_HIGH>;
+ pinctrl-names = "default";
+ pinctrl-0 = <&usb3drd_boost_en>;
+ enable-active-high;
+ };
+
};
&fin_pll {
&pinctrl_alive {
pmic_irq: pmic-irq {
samsung,pins = "gpa0-2";
- samsung,pin-pud = <3>;
- samsung,pin-drv = <3>;
+ samsung,pin-pud = <EXYNOS_PIN_PULL_UP>;
+ samsung,pin-drv = <EXYNOS4_PIN_DRV_LV4>;
};
};
vqmmc-supply = <&ldo2_reg>;
disable-wp;
};
+
+&pinctrl_bus1 {
+ usb30_vbus_en: usb30-vbus-en {
+ samsung,pins = "gph1-1";
+ samsung,pin-function = <EXYNOS_PIN_FUNC_OUTPUT>;
+ samsung,pin-pud = <EXYNOS_PIN_PULL_NONE>;
+ samsung,pin-drv = <EXYNOS4_PIN_DRV_LV1>;
+ };
+
+ usb3drd_boost_en: usb3drd-boost-en {
+ samsung,pins = "gpf4-1";
+ samsung,pin-function = <EXYNOS_PIN_FUNC_OUTPUT>;
+ samsung,pin-pud = <EXYNOS_PIN_PULL_NONE>;
+ samsung,pin-drv = <EXYNOS4_PIN_DRV_LV1>;
+ };
+};
+
+&usbdrd_phy {
+ vbus-supply = <&usb30_vbus_reg>;
+ vbus-boost-supply = <&usb3drd_boost_5v>;
+};
* published by the Free Software Foundation.
*/
+#include <dt-bindings/pinctrl/samsung.h>
+
&pinctrl_alive {
gpa0: gpa0 {
gpio-controller;
hs_i2c10_bus: hs-i2c10-bus {
samsung,pins = "gpb0-1", "gpb0-0";
- samsung,pin-function = <2>;
- samsung,pin-pud = <3>;
- samsung,pin-drv = <0>;
+ samsung,pin-function = <EXYNOS_PIN_FUNC_2>;
+ samsung,pin-pud = <EXYNOS_PIN_PULL_UP>;
+ samsung,pin-drv = <EXYNOS4_PIN_DRV_LV1>;
};
hs_i2c11_bus: hs-i2c11-bus {
samsung,pins = "gpb0-3", "gpb0-2";
- samsung,pin-function = <2>;
- samsung,pin-pud = <3>;
- samsung,pin-drv = <0>;
+ samsung,pin-function = <EXYNOS_PIN_FUNC_2>;
+ samsung,pin-pud = <EXYNOS_PIN_PULL_UP>;
+ samsung,pin-drv = <EXYNOS4_PIN_DRV_LV1>;
};
hs_i2c2_bus: hs-i2c2-bus {
samsung,pins = "gpd0-3", "gpd0-2";
- samsung,pin-function = <3>;
- samsung,pin-pud = <3>;
- samsung,pin-drv = <0>;
+ samsung,pin-function = <EXYNOS_PIN_FUNC_3>;
+ samsung,pin-pud = <EXYNOS_PIN_PULL_UP>;
+ samsung,pin-drv = <EXYNOS4_PIN_DRV_LV1>;
};
uart0_data: uart0-data {
samsung,pins = "gpd0-0", "gpd0-1";
- samsung,pin-function = <2>;
- samsung,pin-pud = <0>;
- samsung,pin-drv = <0>;
+ samsung,pin-function = <EXYNOS_PIN_FUNC_2>;
+ samsung,pin-pud = <EXYNOS_PIN_PULL_NONE>;
+ samsung,pin-drv = <EXYNOS4_PIN_DRV_LV1>;
};
uart0_fctl: uart0-fctl {
samsung,pins = "gpd0-2", "gpd0-3";
- samsung,pin-function = <2>;
- samsung,pin-pud = <0>;
- samsung,pin-drv = <0>;
+ samsung,pin-function = <EXYNOS_PIN_FUNC_2>;
+ samsung,pin-pud = <EXYNOS_PIN_PULL_NONE>;
+ samsung,pin-drv = <EXYNOS4_PIN_DRV_LV1>;
};
uart2_data: uart2-data {
samsung,pins = "gpd1-4", "gpd1-5";
- samsung,pin-function = <2>;
- samsung,pin-pud = <0>;
- samsung,pin-drv = <0>;
+ samsung,pin-function = <EXYNOS_PIN_FUNC_2>;
+ samsung,pin-pud = <EXYNOS_PIN_PULL_NONE>;
+ samsung,pin-drv = <EXYNOS4_PIN_DRV_LV1>;
};
hs_i2c3_bus: hs-i2c3-bus {
samsung,pins = "gpd1-3", "gpd1-2";
- samsung,pin-function = <3>;
- samsung,pin-pud = <3>;
- samsung,pin-drv = <0>;
+ samsung,pin-function = <EXYNOS_PIN_FUNC_3>;
+ samsung,pin-pud = <EXYNOS_PIN_PULL_UP>;
+ samsung,pin-drv = <EXYNOS4_PIN_DRV_LV1>;
};
uart1_data: uart1-data {
samsung,pins = "gpd1-0", "gpd1-1";
- samsung,pin-function = <2>;
- samsung,pin-pud = <0>;
- samsung,pin-drv = <0>;
+ samsung,pin-function = <EXYNOS_PIN_FUNC_2>;
+ samsung,pin-pud = <EXYNOS_PIN_PULL_NONE>;
+ samsung,pin-drv = <EXYNOS4_PIN_DRV_LV1>;
};
uart1_fctl: uart1-fctl {
samsung,pins = "gpd1-2", "gpd1-3";
- samsung,pin-function = <2>;
- samsung,pin-pud = <0>;
- samsung,pin-drv = <0>;
+ samsung,pin-function = <EXYNOS_PIN_FUNC_2>;
+ samsung,pin-pud = <EXYNOS_PIN_PULL_NONE>;
+ samsung,pin-drv = <EXYNOS4_PIN_DRV_LV1>;
};
hs_i2c0_bus: hs-i2c0-bus {
samsung,pins = "gpd2-1", "gpd2-0";
- samsung,pin-function = <2>;
- samsung,pin-pud = <3>;
- samsung,pin-drv = <0>;
+ samsung,pin-function = <EXYNOS_PIN_FUNC_2>;
+ samsung,pin-pud = <EXYNOS_PIN_PULL_UP>;
+ samsung,pin-drv = <EXYNOS4_PIN_DRV_LV1>;
};
hs_i2c1_bus: hs-i2c1-bus {
samsung,pins = "gpd2-3", "gpd2-2";
- samsung,pin-function = <2>;
- samsung,pin-pud = <3>;
- samsung,pin-drv = <0>;
+ samsung,pin-function = <EXYNOS_PIN_FUNC_2>;
+ samsung,pin-pud = <EXYNOS_PIN_PULL_UP>;
+ samsung,pin-drv = <EXYNOS4_PIN_DRV_LV1>;
};
hs_i2c9_bus: hs-i2c9-bus {
samsung,pins = "gpd2-7", "gpd2-6";
- samsung,pin-function = <3>;
- samsung,pin-pud = <3>;
- samsung,pin-drv = <0>;
+ samsung,pin-function = <EXYNOS_PIN_FUNC_3>;
+ samsung,pin-pud = <EXYNOS_PIN_PULL_UP>;
+ samsung,pin-drv = <EXYNOS4_PIN_DRV_LV1>;
};
pwm0_out: pwm0-out {
samsung,pins = "gpd2-4";
- samsung,pin-function = <2>;
- samsung,pin-pud = <0>;
- samsung,pin-drv = <0>;
+ samsung,pin-function = <EXYNOS_PIN_FUNC_2>;
+ samsung,pin-pud = <EXYNOS_PIN_PULL_NONE>;
+ samsung,pin-drv = <EXYNOS4_PIN_DRV_LV1>;
};
pwm1_out: pwm1-out {
samsung,pins = "gpd2-5";
- samsung,pin-function = <2>;
- samsung,pin-pud = <0>;
- samsung,pin-drv = <0>;
+ samsung,pin-function = <EXYNOS_PIN_FUNC_2>;
+ samsung,pin-pud = <EXYNOS_PIN_PULL_NONE>;
+ samsung,pin-drv = <EXYNOS4_PIN_DRV_LV1>;
};
pwm2_out: pwm2-out {
samsung,pins = "gpd2-6";
- samsung,pin-function = <2>;
- samsung,pin-pud = <0>;
- samsung,pin-drv = <0>;
+ samsung,pin-function = <EXYNOS_PIN_FUNC_2>;
+ samsung,pin-pud = <EXYNOS_PIN_PULL_NONE>;
+ samsung,pin-drv = <EXYNOS4_PIN_DRV_LV1>;
};
pwm3_out: pwm3-out {
samsung,pins = "gpd2-7";
- samsung,pin-function = <2>;
- samsung,pin-pud = <0>;
- samsung,pin-drv = <0>;
+ samsung,pin-function = <EXYNOS_PIN_FUNC_2>;
+ samsung,pin-pud = <EXYNOS_PIN_PULL_NONE>;
+ samsung,pin-drv = <EXYNOS4_PIN_DRV_LV1>;
};
hs_i2c8_bus: hs-i2c8-bus {
samsung,pins = "gpd5-3", "gpd5-2";
- samsung,pin-function = <3>;
- samsung,pin-pud = <3>;
- samsung,pin-drv = <0>;
+ samsung,pin-function = <EXYNOS_PIN_FUNC_3>;
+ samsung,pin-pud = <EXYNOS_PIN_PULL_UP>;
+ samsung,pin-drv = <EXYNOS4_PIN_DRV_LV1>;
};
uart3_data: uart3-data {
samsung,pins = "gpd5-0", "gpd5-1";
- samsung,pin-function = <3>;
- samsung,pin-pud = <0>;
- samsung,pin-drv = <0>;
+ samsung,pin-function = <EXYNOS_PIN_FUNC_3>;
+ samsung,pin-pud = <EXYNOS_PIN_PULL_NONE>;
+ samsung,pin-drv = <EXYNOS4_PIN_DRV_LV1>;
};
spi2_bus: spi2-bus {
samsung,pins = "gpd5-0", "gpd5-1", "gpd5-2", "gpd5-3";
- samsung,pin-function = <2>;
- samsung,pin-pud = <3>;
- samsung,pin-drv = <0>;
+ samsung,pin-function = <EXYNOS_PIN_FUNC_2>;
+ samsung,pin-pud = <EXYNOS_PIN_PULL_UP>;
+ samsung,pin-drv = <EXYNOS4_PIN_DRV_LV1>;
};
spi1_bus: spi1-bus {
samsung,pins = "gpd6-2", "gpd6-3", "gpd6-4", "gpd6-5";
- samsung,pin-function = <2>;
- samsung,pin-pud = <3>;
- samsung,pin-drv = <0>;
+ samsung,pin-function = <EXYNOS_PIN_FUNC_2>;
+ samsung,pin-pud = <EXYNOS_PIN_PULL_UP>;
+ samsung,pin-drv = <EXYNOS4_PIN_DRV_LV1>;
};
spi0_bus: spi0-bus {
samsung,pins = "gpd8-0", "gpd8-1", "gpd6-0", "gpd6-1";
- samsung,pin-function = <2>;
- samsung,pin-pud = <3>;
- samsung,pin-drv = <0>;
+ samsung,pin-function = <EXYNOS_PIN_FUNC_2>;
+ samsung,pin-pud = <EXYNOS_PIN_PULL_UP>;
+ samsung,pin-drv = <EXYNOS4_PIN_DRV_LV1>;
};
hs_i2c4_bus: hs-i2c4-bus {
samsung,pins = "gpg3-1", "gpg3-0";
- samsung,pin-function = <2>;
- samsung,pin-pud = <3>;
- samsung,pin-drv = <0>;
+ samsung,pin-function = <EXYNOS_PIN_FUNC_2>;
+ samsung,pin-pud = <EXYNOS_PIN_PULL_UP>;
+ samsung,pin-drv = <EXYNOS4_PIN_DRV_LV1>;
};
hs_i2c5_bus: hs-i2c5-bus {
samsung,pins = "gpg3-3", "gpg3-2";
- samsung,pin-function = <2>;
- samsung,pin-pud = <3>;
- samsung,pin-drv = <0>;
+ samsung,pin-function = <EXYNOS_PIN_FUNC_2>;
+ samsung,pin-pud = <EXYNOS_PIN_PULL_UP>;
+ samsung,pin-drv = <EXYNOS4_PIN_DRV_LV1>;
};
};
hs_i2c6_bus: hs-i2c6-bus {
samsung,pins = "gpj0-1", "gpj0-0";
- samsung,pin-function = <2>;
- samsung,pin-pud = <3>;
- samsung,pin-drv = <0>;
+ samsung,pin-function = <EXYNOS_PIN_FUNC_2>;
+ samsung,pin-pud = <EXYNOS_PIN_PULL_UP>;
+ samsung,pin-drv = <EXYNOS4_PIN_DRV_LV1>;
};
};
hs_i2c7_bus: hs-i2c7-bus {
samsung,pins = "gpj1-1", "gpj1-0";
- samsung,pin-function = <2>;
- samsung,pin-pud = <3>;
- samsung,pin-drv = <0>;
+ samsung,pin-function = <EXYNOS_PIN_FUNC_2>;
+ samsung,pin-pud = <EXYNOS_PIN_PULL_UP>;
+ samsung,pin-drv = <EXYNOS4_PIN_DRV_LV1>;
};
};
spi3_bus: spi3-bus {
samsung,pins = "gpg4-0", "gpg4-1", "gpg4-2", "gpg4-3";
- samsung,pin-function = <2>;
- samsung,pin-pud = <3>;
- samsung,pin-drv = <0>;
+ samsung,pin-function = <EXYNOS_PIN_FUNC_2>;
+ samsung,pin-pud = <EXYNOS_PIN_PULL_UP>;
+ samsung,pin-drv = <EXYNOS4_PIN_DRV_LV1>;
};
};
spi4_bus: spi4-bus {
samsung,pins = "gpv7-0", "gpv7-1", "gpv7-2", "gpv7-3";
- samsung,pin-function = <2>;
- samsung,pin-pud = <3>;
- samsung,pin-drv = <0>;
+ samsung,pin-function = <EXYNOS_PIN_FUNC_2>;
+ samsung,pin-pud = <EXYNOS_PIN_PULL_UP>;
+ samsung,pin-drv = <EXYNOS4_PIN_DRV_LV1>;
};
};
sd2_clk: sd2-clk {
samsung,pins = "gpr4-0";
- samsung,pin-function = <2>;
- samsung,pin-pud = <0>;
- samsung,pin-drv = <3>;
+ samsung,pin-function = <EXYNOS_PIN_FUNC_2>;
+ samsung,pin-pud = <EXYNOS_PIN_PULL_NONE>;
+ samsung,pin-drv = <EXYNOS4_PIN_DRV_LV4>;
};
sd2_cmd: sd2-cmd {
samsung,pins = "gpr4-1";
- samsung,pin-function = <2>;
- samsung,pin-pud = <0>;
- samsung,pin-drv = <3>;
+ samsung,pin-function = <EXYNOS_PIN_FUNC_2>;
+ samsung,pin-pud = <EXYNOS_PIN_PULL_NONE>;
+ samsung,pin-drv = <EXYNOS4_PIN_DRV_LV4>;
};
sd2_cd: sd2-cd {
samsung,pins = "gpr4-2";
- samsung,pin-function = <2>;
- samsung,pin-pud = <3>;
- samsung,pin-drv = <3>;
+ samsung,pin-function = <EXYNOS_PIN_FUNC_2>;
+ samsung,pin-pud = <EXYNOS_PIN_PULL_UP>;
+ samsung,pin-drv = <EXYNOS4_PIN_DRV_LV4>;
};
sd2_bus1: sd2-bus-width1 {
samsung,pins = "gpr4-3";
- samsung,pin-function = <2>;
- samsung,pin-pud = <3>;
- samsung,pin-drv = <3>;
+ samsung,pin-function = <EXYNOS_PIN_FUNC_2>;
+ samsung,pin-pud = <EXYNOS_PIN_PULL_UP>;
+ samsung,pin-drv = <EXYNOS4_PIN_DRV_LV4>;
};
sd2_bus4: sd2-bus-width4 {
samsung,pins = "gpr4-4", "gpr4-5", "gpr4-6";
- samsung,pin-function = <2>;
- samsung,pin-pud = <3>;
- samsung,pin-drv = <3>;
+ samsung,pin-function = <EXYNOS_PIN_FUNC_2>;
+ samsung,pin-pud = <EXYNOS_PIN_PULL_UP>;
+ samsung,pin-drv = <EXYNOS4_PIN_DRV_LV4>;
};
};
sd0_clk: sd0-clk {
samsung,pins = "gpr0-0";
- samsung,pin-function = <2>;
- samsung,pin-pud = <0>;
- samsung,pin-drv = <4>;
+ samsung,pin-function = <EXYNOS_PIN_FUNC_2>;
+ samsung,pin-pud = <EXYNOS_PIN_PULL_NONE>;
+ samsung,pin-drv = <EXYNOS7_FSYS1_PIN_DRV_LV2>;
};
sd0_cmd: sd0-cmd {
samsung,pins = "gpr0-1";
- samsung,pin-function = <2>;
- samsung,pin-pud = <3>;
- samsung,pin-drv = <4>;
+ samsung,pin-function = <EXYNOS_PIN_FUNC_2>;
+ samsung,pin-pud = <EXYNOS_PIN_PULL_UP>;
+ samsung,pin-drv = <EXYNOS7_FSYS1_PIN_DRV_LV2>;
};
sd0_ds: sd0-ds {
samsung,pins = "gpr0-2";
- samsung,pin-function = <2>;
- samsung,pin-pud = <1>;
- samsung,pin-drv = <4>;
+ samsung,pin-function = <EXYNOS_PIN_FUNC_2>;
+ samsung,pin-pud = <EXYNOS_PIN_PULL_DOWN>;
+ samsung,pin-drv = <EXYNOS7_FSYS1_PIN_DRV_LV2>;
};
sd0_qrdy: sd0-qrdy {
samsung,pins = "gpr0-3";
- samsung,pin-function = <2>;
- samsung,pin-pud = <1>;
- samsung,pin-drv = <4>;
+ samsung,pin-function = <EXYNOS_PIN_FUNC_2>;
+ samsung,pin-pud = <EXYNOS_PIN_PULL_DOWN>;
+ samsung,pin-drv = <EXYNOS7_FSYS1_PIN_DRV_LV2>;
};
sd0_bus1: sd0-bus-width1 {
samsung,pins = "gpr1-0";
- samsung,pin-function = <2>;
- samsung,pin-pud = <3>;
- samsung,pin-drv = <4>;
+ samsung,pin-function = <EXYNOS_PIN_FUNC_2>;
+ samsung,pin-pud = <EXYNOS_PIN_PULL_UP>;
+ samsung,pin-drv = <EXYNOS7_FSYS1_PIN_DRV_LV2>;
};
sd0_bus4: sd0-bus-width4 {
samsung,pins = "gpr1-1", "gpr1-2", "gpr1-3";
- samsung,pin-function = <2>;
- samsung,pin-pud = <3>;
- samsung,pin-drv = <4>;
+ samsung,pin-function = <EXYNOS_PIN_FUNC_2>;
+ samsung,pin-pud = <EXYNOS_PIN_PULL_UP>;
+ samsung,pin-drv = <EXYNOS7_FSYS1_PIN_DRV_LV2>;
};
sd0_bus8: sd0-bus-width8 {
samsung,pins = "gpr1-4", "gpr1-5", "gpr1-6", "gpr1-7";
- samsung,pin-function = <2>;
- samsung,pin-pud = <3>;
- samsung,pin-drv = <4>;
+ samsung,pin-function = <EXYNOS_PIN_FUNC_2>;
+ samsung,pin-pud = <EXYNOS_PIN_PULL_UP>;
+ samsung,pin-drv = <EXYNOS7_FSYS1_PIN_DRV_LV2>;
};
sd1_clk: sd1-clk {
samsung,pins = "gpr2-0";
- samsung,pin-function = <2>;
- samsung,pin-pud = <0>;
- samsung,pin-drv = <2>;
+ samsung,pin-function = <EXYNOS_PIN_FUNC_2>;
+ samsung,pin-pud = <EXYNOS_PIN_PULL_NONE>;
+ samsung,pin-drv = <EXYNOS7_FSYS1_PIN_DRV_LV3>;
};
sd1_cmd: sd1-cmd {
samsung,pins = "gpr2-1";
- samsung,pin-function = <2>;
- samsung,pin-pud = <0>;
- samsung,pin-drv = <2>;
+ samsung,pin-function = <EXYNOS_PIN_FUNC_2>;
+ samsung,pin-pud = <EXYNOS_PIN_PULL_NONE>;
+ samsung,pin-drv = <EXYNOS7_FSYS1_PIN_DRV_LV3>;
};
sd1_ds: sd1-ds {
samsung,pins = "gpr2-2";
- samsung,pin-function = <2>;
- samsung,pin-pud = <1>;
- samsung,pin-drv = <6>;
+ samsung,pin-function = <EXYNOS_PIN_FUNC_2>;
+ samsung,pin-pud = <EXYNOS_PIN_PULL_DOWN>;
+ samsung,pin-drv = <EXYNOS7_FSYS1_PIN_DRV_LV4>;
};
sd1_qrdy: sd1-qrdy {
samsung,pins = "gpr2-3";
- samsung,pin-function = <2>;
- samsung,pin-pud = <1>;
- samsung,pin-drv = <6>;
+ samsung,pin-function = <EXYNOS_PIN_FUNC_2>;
+ samsung,pin-pud = <EXYNOS_PIN_PULL_DOWN>;
+ samsung,pin-drv = <EXYNOS7_FSYS1_PIN_DRV_LV4>;
};
sd1_int: sd1-int {
samsung,pins = "gpr2-4";
- samsung,pin-function = <2>;
- samsung,pin-pud = <1>;
- samsung,pin-drv = <6>;
+ samsung,pin-function = <EXYNOS_PIN_FUNC_2>;
+ samsung,pin-pud = <EXYNOS_PIN_PULL_DOWN>;
+ samsung,pin-drv = <EXYNOS7_FSYS1_PIN_DRV_LV4>;
};
sd1_bus1: sd1-bus-width1 {
samsung,pins = "gpr3-0";
- samsung,pin-function = <2>;
- samsung,pin-pud = <3>;
- samsung,pin-drv = <2>;
+ samsung,pin-function = <EXYNOS_PIN_FUNC_2>;
+ samsung,pin-pud = <EXYNOS_PIN_PULL_UP>;
+ samsung,pin-drv = <EXYNOS7_FSYS1_PIN_DRV_LV3>;
};
sd1_bus4: sd1-bus-width4 {
samsung,pins = "gpr3-1", "gpr3-2", "gpr3-3";
- samsung,pin-function = <2>;
- samsung,pin-pud = <3>;
- samsung,pin-drv = <2>;
+ samsung,pin-function = <EXYNOS_PIN_FUNC_2>;
+ samsung,pin-pud = <EXYNOS_PIN_PULL_UP>;
+ samsung,pin-drv = <EXYNOS7_FSYS1_PIN_DRV_LV3>;
};
sd1_bus8: sd1-bus-width8 {
samsung,pins = "gpr3-4", "gpr3-5", "gpr3-6", "gpr3-7";
- samsung,pin-function = <2>;
- samsung,pin-pud = <3>;
- samsung,pin-drv = <2>;
+ samsung,pin-function = <EXYNOS_PIN_FUNC_2>;
+ samsung,pin-pud = <EXYNOS_PIN_PULL_UP>;
+ samsung,pin-drv = <EXYNOS7_FSYS1_PIN_DRV_LV3>;
};
};
spi5_bus: spi5-bus {
samsung,pins = "gpf2-0", "gpf2-1", "gpf2-2", "gpf2-3";
- samsung,pin-function = <2>;
- samsung,pin-pud = <3>;
- samsung,pin-drv = <0>;
+ samsung,pin-function = <EXYNOS_PIN_FUNC_2>;
+ samsung,pin-pud = <EXYNOS_PIN_PULL_UP>;
+ samsung,pin-drv = <EXYNOS4_PIN_DRV_LV1>;
};
ufs_refclk_out: ufs-refclk-out {
samsung,pins = "gpg2-4";
- samsung,pin-function = <2>;
- samsung,pin-pud = <0>;
- samsung,pin-drv = <2>;
+ samsung,pin-function = <EXYNOS_PIN_FUNC_2>;
+ samsung,pin-pud = <EXYNOS_PIN_PULL_NONE>;
+ samsung,pin-drv = <EXYNOS4_PIN_DRV_LV2>;
};
ufs_rst_n: ufs-rst-n {
samsung,pins = "gph1-5";
- samsung,pin-function = <2>;
- samsung,pin-pud = <3>;
- samsung,pin-drv = <0>;
+ samsung,pin-function = <EXYNOS_PIN_FUNC_2>;
+ samsung,pin-pud = <EXYNOS_PIN_PULL_UP>;
+ samsung,pin-drv = <EXYNOS4_PIN_DRV_LV1>;
};
};
#include "exynos7-trip-points.dtsi"
};
};
+
+ usbdrd_phy: phy@15500000 {
+ compatible = "samsung,exynos7-usbdrd-phy";
+ reg = <0x15500000 0x100>;
+ clocks = <&clock_fsys0 ACLK_USBDRD300>,
+ <&clock_fsys0 OSCCLK_PHY_CLKOUT_USB30_PHY>,
+ <&clock_fsys0 PHYCLK_USBDRD300_UDRD30_PIPE_PCLK_USER>,
+ <&clock_fsys0 PHYCLK_USBDRD300_UDRD30_PHYCLK_USER>,
+ <&clock_fsys0 SCLK_USBDRD300_REFCLK>;
+ clock-names = "phy", "ref", "phy_pipe",
+ "phy_utmi", "itp";
+ samsung,pmu-syscon = <&pmu_system_controller>;
+ #phy-cells = <1>;
+ };
+
+ usbdrd3 {
+ compatible = "samsung,exynos7-dwusb3";
+ clocks = <&clock_fsys0 ACLK_USBDRD300>,
+ <&clock_fsys0 SCLK_USBDRD300_SUSPENDCLK>,
+ <&clock_fsys0 ACLK_AXIUS_USBDRD30X_FSYS0X>;
+ clock-names = "usbdrd30", "usbdrd30_susp_clk",
+ "usbdrd30_axius_clk";
+ #address-cells = <1>;
+ #size-cells = <1>;
+ ranges;
+
+ dwc3@15400000 {
+ compatible = "snps,dwc3";
+ reg = <0x15400000 0x10000>;
+ interrupts = <GIC_SPI 223 IRQ_TYPE_LEVEL_HIGH>;
+ phys = <&usbdrd_phy 0>, <&usbdrd_phy 1>;
+ phy-names = "usb2-phy", "usb3-phy";
+ };
+ };
};
};
*/
#include <linux/types.h>
#include <linux/sched.h>
+#include <linux/sched/task_stack.h>
#define COMPAT_USER_HZ 100
#ifdef __AARCH64EB__
#define __KVM_HAVE_ARCH_INTC_INITIALIZED
-#define KVM_USER_MEM_SLOTS 32
-#define KVM_PRIVATE_MEM_SLOTS 4
+#define KVM_USER_MEM_SLOTS 512
#define KVM_COALESCED_MMIO_PAGE_OFFSET 1
#define KVM_HALT_POLL_NS_DEFAULT 500000
#include <linux/compiler.h>
#include <linux/sched.h>
+#include <linux/sched/hotplug.h>
+#include <linux/mm_types.h>
#include <asm/cacheflush.h>
#include <asm/cpufeature.h>
#define __pgprot(x) ((pgprot_t) { (x) } )
#if CONFIG_PGTABLE_LEVELS == 2
+#define __ARCH_USE_5LEVEL_HACK
#include <asm-generic/pgtable-nopmd.h>
#elif CONFIG_PGTABLE_LEVELS == 3
+#define __ARCH_USE_5LEVEL_HACK
#include <asm-generic/pgtable-nopud.h>
+#elif CONFIG_PGTABLE_LEVELS == 4
+#include <asm-generic/5level-fixup.h>
#endif
#endif /* __ASM_PGTABLE_TYPES_H */
#include <linux/kprobes.h>
#include <linux/stat.h>
#include <linux/uaccess.h>
+#include <linux/sched/task_stack.h>
#include <asm/cpufeature.h>
#include <asm/cputype.h>
#include <linux/cpu_pm.h>
#include <linux/kernel.h>
#include <linux/init.h>
-#include <linux/sched.h>
+#include <linux/sched/signal.h>
#include <linux/signal.h>
#include <linux/hardirq.h>
#include <linux/kdebug.h>
#include <linux/kgdb.h>
#include <linux/kprobes.h>
+#include <linux/sched/task_stack.h>
+
#include <asm/debug-monitors.h>
#include <asm/insn.h>
#include <asm/traps.h>
#include <linux/kernel.h>
#include <linux/perf_event.h>
#include <linux/bug.h>
+#include <linux/sched/task_stack.h>
#include <asm/compat.h>
#include <asm/perf_regs.h>
#include <linux/extable.h>
#include <linux/slab.h>
#include <linux/stop_machine.h>
+#include <linux/sched/debug.h>
#include <linux/stringify.h>
#include <asm/traps.h>
#include <asm/ptrace.h>
#include <linux/efi.h>
#include <linux/export.h>
#include <linux/sched.h>
+#include <linux/sched/debug.h>
+#include <linux/sched/task.h>
+#include <linux/sched/task_stack.h>
#include <linux/kernel.h>
#include <linux/mm.h>
#include <linux/stddef.h>
#include <linux/audit.h>
#include <linux/compat.h>
#include <linux/kernel.h>
-#include <linux/sched.h>
+#include <linux/sched/signal.h>
+#include <linux/sched/task_stack.h>
#include <linux/mm.h>
#include <linux/smp.h>
#include <linux/ptrace.h>
#include <linux/of_fdt.h>
#include <linux/efi.h>
#include <linux/psci.h>
+#include <linux/sched/task.h>
#include <linux/mm.h>
#include <asm/acpi.h>
#include <linux/delay.h>
#include <linux/init.h>
#include <linux/spinlock.h>
-#include <linux/sched.h>
+#include <linux/sched/mm.h>
+#include <linux/sched/hotplug.h>
+#include <linux/sched/task_stack.h>
#include <linux/interrupt.h>
#include <linux/cache.h>
#include <linux/profile.h>
#include <linux/export.h>
#include <linux/ftrace.h>
#include <linux/sched.h>
+#include <linux/sched/debug.h>
+#include <linux/sched/task_stack.h>
#include <linux/stacktrace.h>
#include <asm/irq.h>
#include <linux/compat.h>
#include <linux/personality.h>
#include <linux/sched.h>
+#include <linux/sched/signal.h>
#include <linux/slab.h>
#include <linux/syscalls.h>
#include <linux/uaccess.h>
#include <linux/nodemask.h>
#include <linux/of.h>
#include <linux/sched.h>
+#include <linux/sched/topology.h>
#include <linux/slab.h>
#include <linux/string.h>
#include <linux/cpufreq.h>
#include <linux/kexec.h>
#include <linux/delay.h>
#include <linux/init.h>
-#include <linux/sched.h>
+#include <linux/sched/signal.h>
+#include <linux/sched/debug.h>
+#include <linux/sched/task_stack.h>
#include <linux/syscalls.h>
+#include <linux/mm_types.h>
#include <asm/atomic.h>
#include <asm/bug.h>
return ret;
}
+static int kvm_handle_unknown_ec(struct kvm_vcpu *vcpu, struct kvm_run *run)
+{
+ u32 hsr = kvm_vcpu_get_hsr(vcpu);
+
+ kvm_pr_unimpl("Unknown exception class: hsr: %#08x -- %s\n",
+ hsr, esr_get_class_string(hsr));
+
+ kvm_inject_undefined(vcpu);
+ return 1;
+}
+
static exit_handle_fn arm_exit_handlers[] = {
+ [0 ... ESR_ELx_EC_MAX] = kvm_handle_unknown_ec,
[ESR_ELx_EC_WFx] = kvm_handle_wfx,
[ESR_ELx_EC_CP15_32] = kvm_handle_cp15_32,
[ESR_ELx_EC_CP15_64] = kvm_handle_cp15_64,
u32 hsr = kvm_vcpu_get_hsr(vcpu);
u8 hsr_ec = ESR_ELx_EC(hsr);
- if (hsr_ec >= ARRAY_SIZE(arm_exit_handlers) ||
- !arm_exit_handlers[hsr_ec]) {
- kvm_err("Unknown exception class: hsr: %#08x -- %s\n",
- hsr, esr_get_class_string(hsr));
- BUG();
- }
-
return arm_exit_handlers[hsr_ec];
}
#include <asm/kvm_hyp.h>
#include <asm/tlbflush.h>
+static void __hyp_text __tlb_switch_to_guest_vhe(struct kvm *kvm)
+{
+ u64 val;
+
+ /*
+ * With VHE enabled, we have HCR_EL2.{E2H,TGE} = {1,1}, and
+ * most TLB operations target EL2/EL0. In order to affect the
+ * guest TLBs (EL1/EL0), we need to change one of these two
+ * bits. Changing E2H is impossible (goodbye TTBR1_EL2), so
+ * let's flip TGE before executing the TLB operation.
+ */
+ write_sysreg(kvm->arch.vttbr, vttbr_el2);
+ val = read_sysreg(hcr_el2);
+ val &= ~HCR_TGE;
+ write_sysreg(val, hcr_el2);
+ isb();
+}
+
+static void __hyp_text __tlb_switch_to_guest_nvhe(struct kvm *kvm)
+{
+ write_sysreg(kvm->arch.vttbr, vttbr_el2);
+ isb();
+}
+
+static hyp_alternate_select(__tlb_switch_to_guest,
+ __tlb_switch_to_guest_nvhe,
+ __tlb_switch_to_guest_vhe,
+ ARM64_HAS_VIRT_HOST_EXTN);
+
+static void __hyp_text __tlb_switch_to_host_vhe(struct kvm *kvm)
+{
+ /*
+ * We're done with the TLB operation, let's restore the host's
+ * view of HCR_EL2.
+ */
+ write_sysreg(0, vttbr_el2);
+ write_sysreg(HCR_HOST_VHE_FLAGS, hcr_el2);
+}
+
+static void __hyp_text __tlb_switch_to_host_nvhe(struct kvm *kvm)
+{
+ write_sysreg(0, vttbr_el2);
+}
+
+static hyp_alternate_select(__tlb_switch_to_host,
+ __tlb_switch_to_host_nvhe,
+ __tlb_switch_to_host_vhe,
+ ARM64_HAS_VIRT_HOST_EXTN);
+
void __hyp_text __kvm_tlb_flush_vmid_ipa(struct kvm *kvm, phys_addr_t ipa)
{
dsb(ishst);
/* Switch to requested VMID */
kvm = kern_hyp_va(kvm);
- write_sysreg(kvm->arch.vttbr, vttbr_el2);
- isb();
+ __tlb_switch_to_guest()(kvm);
/*
* We could do so much better if we had the VA as well.
dsb(ish);
isb();
- write_sysreg(0, vttbr_el2);
+ __tlb_switch_to_host()(kvm);
}
void __hyp_text __kvm_tlb_flush_vmid(struct kvm *kvm)
/* Switch to requested VMID */
kvm = kern_hyp_va(kvm);
- write_sysreg(kvm->arch.vttbr, vttbr_el2);
- isb();
+ __tlb_switch_to_guest()(kvm);
__tlbi(vmalls12e1is);
dsb(ish);
isb();
- write_sysreg(0, vttbr_el2);
+ __tlb_switch_to_host()(kvm);
}
void __hyp_text __kvm_tlb_flush_local_vmid(struct kvm_vcpu *vcpu)
struct kvm *kvm = kern_hyp_va(kern_hyp_va(vcpu)->kvm);
/* Switch to requested VMID */
- write_sysreg(kvm->arch.vttbr, vttbr_el2);
- isb();
+ __tlb_switch_to_guest()(kvm);
__tlbi(vmalle1);
dsb(nsh);
isb();
- write_sysreg(0, vttbr_el2);
+ __tlb_switch_to_host()(kvm);
}
void __hyp_text __kvm_flush_vm_context(void)
#include <linux/kprobes.h>
#include <linux/uaccess.h>
#include <linux/page-flags.h>
-#include <linux/sched.h>
+#include <linux/sched/signal.h>
+#include <linux/sched/debug.h>
#include <linux/highmem.h>
#include <linux/perf_event.h>
#include <linux/preempt.h>
#define pr_fmt(fmt) "kasan: " fmt
#include <linux/kasan.h>
#include <linux/kernel.h>
+#include <linux/sched/task.h>
#include <linux/memblock.h>
#include <linux/start_kernel.h>
#include <linux/mm.h>
#include <linux/mman.h>
#include <linux/export.h>
#include <linux/shm.h>
-#include <linux/sched.h>
+#include <linux/sched/signal.h>
+#include <linux/sched/mm.h>
#include <linux/io.h>
#include <linux/personality.h>
#include <linux/random.h>
#ifndef __ASM_AVR32_MMU_CONTEXT_H
#define __ASM_AVR32_MMU_CONTEXT_H
+#include <linux/mm_types.h>
+
#include <asm/tlbflush.h>
#include <asm/sysreg.h>
#include <asm-generic/mm_hooks.h>
#ifndef __ASM_AVR32_PGTABLE_2LEVEL_H
#define __ASM_AVR32_PGTABLE_2LEVEL_H
+#define __ARCH_USE_5LEVEL_HACK
#include <asm-generic/pgtable-nopmd.h>
/*
#include <linux/kdebug.h>
#include <linux/notifier.h>
#include <linux/sched.h>
+#include <linux/sched/debug.h>
#include <asm/irq.h>
* published by the Free Software Foundation.
*/
#include <linux/sched.h>
+#include <linux/sched/debug.h>
+#include <linux/sched/task.h>
+#include <linux/sched/task_stack.h>
#include <linux/module.h>
#include <linux/kallsyms.h>
#include <linux/fs.h>
#undef DEBUG
#include <linux/kernel.h>
#include <linux/sched.h>
+#include <linux/sched/task_stack.h>
#include <linux/mm.h>
#include <linux/ptrace.h>
#include <linux/errno.h>
* published by the Free Software Foundation.
*/
#include <linux/sched.h>
+#include <linux/sched/task_stack.h>
#include <linux/stacktrace.h>
#include <linux/thread_info.h>
#include <linux/module.h>
#include <linux/extable.h>
#include <linux/module.h> /* print_modules */
#include <linux/notifier.h>
-#include <linux/sched.h>
+#include <linux/sched/signal.h>
#include <linux/uaccess.h>
#include <asm/addrspace.h>
*/
#include <linux/oprofile.h>
-#include <linux/sched.h>
+#include <linux/ptrace.h>
#include <linux/uaccess.h>
/* The first two words of each frame on the stack look like this if we have
#include <linux/slab.h>
#include <linux/sched.h>
+#include <linux/mm_types.h>
+
#include <asm/setup.h>
#include <asm/page.h>
#include <asm/pgalloc.h>
#include <linux/mm.h>
#include <linux/uaccess.h>
#include <linux/module.h>
+#include <linux/sched/debug.h>
+
#include <asm/trace.h>
/*
*/
#include <linux/kernel.h>
+#include <linux/sched/debug.h>
#include <linux/init.h>
#include <linux/serial_core.h>
#include <linux/console.h>
#include <linux/module.h>
#include <linux/sched.h>
+#include <linux/mm_types.h>
#include <linux/flat.h>
#define FLAT_BFIN_RELOC_TYPE_16_BIT 0
#include <linux/nmi.h>
#include <linux/smp.h>
#include <linux/timer.h>
+#include <linux/sched/debug.h>
#include <asm/blackfin.h>
#include <linux/atomic.h>
#include <asm/cacheflush.h>
#include <linux/uaccess.h>
#include <linux/slab.h>
#include <linux/sched.h>
+#include <linux/sched/debug.h>
+#include <linux/sched/task.h>
+#include <linux/sched/task_stack.h>
+#include <linux/mm_types.h>
#include <linux/tick.h>
#include <linux/fs.h>
#include <linux/err.h>
#include <linux/kernel.h>
#include <linux/sched.h>
+#include <linux/sched/task_stack.h>
#include <linux/mm.h>
#include <linux/smp.h>
#include <linux/elf.h>
#include <linux/binfmts.h>
#include <linux/uaccess.h>
#include <linux/tracehook.h>
+#include <linux/sched/task_stack.h>
#include <asm/cacheflush.h>
#include <asm/ucontext.h>
*/
#include <linux/sched.h>
+#include <linux/sched/task_stack.h>
#include <linux/stacktrace.h>
#include <linux/thread_info.h>
#include <linux/module.h>
#include <linux/thread_info.h>
#include <linux/mm.h>
#include <linux/oom.h>
-#include <linux/sched.h>
+#include <linux/sched/signal.h>
+#include <linux/sched/debug.h>
+#include <linux/sched/task.h>
#include <linux/uaccess.h>
#include <linux/module.h>
#include <linux/kallsyms.h>
#include <linux/bug.h>
#include <linux/uaccess.h>
#include <linux/module.h>
+#include <linux/sched/signal.h>
+#include <linux/sched/debug.h>
#include <asm/traps.h>
#include <asm/cplb.h>
#include <asm/blackfin.h>
#include <linux/seq_file.h>
#include <linux/irq.h>
#include <linux/sched.h>
+#include <linux/sched/debug.h>
#include <linux/syscore_ops.h>
#include <linux/gpio.h>
#include <asm/delay.h>
#include <linux/delay.h>
#include <linux/init.h>
#include <linux/spinlock.h>
-#include <linux/sched.h>
+#include <linux/sched/mm.h>
+#include <linux/sched/task_stack.h>
#include <linux/interrupt.h>
#include <linux/cache.h>
#include <linux/clockchips.h>
#include <linux/slab.h>
#include <linux/spinlock.h>
#include <linux/sched.h>
+#include <linux/sched/debug.h>
#include <asm/blackfin.h>
#include <asm/dma.h>
#include <linux/spinlock.h>
#include <linux/rtc.h>
#include <linux/slab.h>
+#include <linux/mm_types.h>
+
#include <asm/blackfin.h>
#include <asm/mem_map.h>
#include "blackfin_sram.h"
#include <linux/mqueue.h>
#include <linux/syscalls.h>
#include <linux/reboot.h>
+#include <linux/sched/task.h>
+#include <linux/sched/task_stack.h>
#include <asm/syscalls.h>
#include <linux/tracehook.h>
#include <linux/regset.h>
#include <linux/elf.h>
+#include <linux/sched/task_stack.h>
#include <asm/cacheflush.h>
*/
#include <linux/module.h>
#include <linux/ptrace.h>
+#include <linux/sched/debug.h>
#include <linux/kallsyms.h>
#include <linux/bug.h>
#include <linux/types.h>
#include <linux/errno.h>
#include <linux/major.h>
-#include <linux/sched.h>
+#include <linux/sched/signal.h>
#include <linux/interrupt.h>
#include <linux/poll.h>
#include <linux/init.h>
*/
#include <linux/sched.h>
+#include <linux/sched/debug.h>
+#include <linux/sched/task.h>
+#include <linux/sched/task_stack.h>
#include <linux/slab.h>
#include <linux/err.h>
#include <linux/fs.h>
#include <linux/kernel.h>
#include <linux/sched.h>
+#include <linux/sched/task_stack.h>
#include <linux/mm.h>
#include <linux/smp.h>
#include <linux/errno.h>
*/
#include <linux/sched.h>
+#include <linux/sched/task_stack.h>
#include <linux/mm.h>
#include <linux/smp.h>
#include <linux/kernel.h>
#include <linux/ptrace.h>
#include <linux/uaccess.h>
+#include <linux/sched/debug.h>
+
#include <arch/sv_addr_ag.h>
#include <arch/system.h>
*
*/
+#include <linux/mm_types.h>
+
#include <asm/tlb.h>
#include <asm/mmu_context.h>
#include <arch/svinto.h>
dma_in_cfg.en = regk_dma_no;
REG_WR(dma, IN_DMA_INST, rw_cfg, dma_in_cfg);
- /* Disble the cryptocop. */
+ /* Disable the cryptocop. */
rw_cfg = REG_RD(strcop, regi_strcop, rw_cfg);
rw_cfg.en = 0;
REG_WR(strcop, regi_strcop, rw_cfg, rw_cfg);
#include <linux/types.h>
#include <linux/errno.h>
#include <linux/major.h>
-#include <linux/sched.h>
+#include <linux/sched/signal.h>
#include <linux/mutex.h>
#include <linux/interrupt.h>
#include <linux/poll.h>
*/
#include <linux/sched.h>
+#include <linux/sched/debug.h>
+#include <linux/sched/task.h>
+#include <linux/sched/task_stack.h>
#include <linux/slab.h>
#include <linux/err.h>
#include <linux/fs.h>
#include <linux/kernel.h>
#include <linux/sched.h>
+#include <linux/sched/task_stack.h>
#include <linux/mm.h>
#include <linux/smp.h>
#include <linux/errno.h>
*/
#include <linux/sched.h>
+#include <linux/sched/task_stack.h>
#include <linux/mm.h>
#include <linux/slab.h>
#include <linux/kernel.h>
#include <linux/ptrace.h>
#include <linux/extable.h>
#include <linux/uaccess.h>
+#include <linux/sched/debug.h>
+
#include <hwregs/supp_reg.h>
#include <hwregs/intr_vect_defs.h>
#include <asm/irq.h>
* Authors: Bjorn Wesen <bjornw@axis.com>
* Tobias Anderberg <tobiasa@axis.com>, CRISv32 port.
*/
+#include <linux/mm_types.h>
#include <asm/tlb.h>
#include <asm/mmu_context.h>
#define _CRIS_PGTABLE_H
#include <asm/page.h>
+#define __ARCH_USE_5LEVEL_HACK
#include <asm-generic/pgtable-nopmd.h>
#ifndef __ASSEMBLY__
-#include <linux/sched.h>
+#include <linux/sched/mm.h>
#include <asm/mmu.h>
#endif
#include <arch/pgtable.h>
#include <linux/module.h>
#include <linux/ptrace.h>
#include <linux/irq.h>
+#include <linux/sched/debug.h>
#include <linux/kernel_stat.h>
#include <linux/signal.h>
#include <linux/spinlock.h>
#include <linux/init_task.h>
#include <linux/sched.h>
+#include <linux/sched/task.h>
#include <linux/fs.h>
#include <linux/user.h>
#include <linux/elfcore.h>
#include <linux/sched.h>
-#include <linux/stacktrace.h>
+#include <linux/sched/debug.h>
#include <linux/stacktrace.h>
#include <asm/stacktrace.h>
#include <linux/timex.h>
#include <linux/init.h>
#include <linux/profile.h>
-#include <linux/sched.h> /* just for sched_clock() - funny that */
+#include <linux/sched/clock.h>
#define D(x)
#include <linux/init.h>
#include <linux/module.h>
#include <linux/utsname.h>
+#include <linux/sched/debug.h>
#ifdef CONFIG_KALLSYMS
#include <linux/kallsyms.h>
#endif
#include <linux/interrupt.h>
#include <linux/extable.h>
#include <linux/wait.h>
+#include <linux/sched/signal.h>
#include <linux/uaccess.h>
#include <arch/system.h>
#include <linux/init.h>
#include <linux/kernel.h>
+#include <linux/mm_types.h>
+
#include <asm/tlb.h>
#define D(x)
#ifndef _ASM_PGTABLE_H
#define _ASM_PGTABLE_H
+#include <asm-generic/5level-fixup.h>
#include <asm/mem-layout.h>
#include <asm/setup.h>
#include <asm/processor.h>
#include <linux/module.h>
#include <linux/errno.h>
#include <linux/sched.h>
+#include <linux/sched/debug.h>
+#include <linux/sched/task.h>
+#include <linux/sched/task_stack.h>
#include <linux/kernel.h>
#include <linux/mm.h>
#include <linux/smp.h>
* 2 of the License, or (at your option) any later version.
*/
-#include <linux/sched.h>
+#include <linux/sched/signal.h>
+#include <linux/sched/debug.h>
#include <linux/signal.h>
#include <linux/kernel.h>
#include <linux/mm.h>
*/
#include <linux/sched.h>
+#include <linux/sched/mm.h>
#include <linux/mm.h>
#include <linux/fs.h>
#include <linux/elf-fdpic.h>
#include <linux/signal.h>
#include <linux/sched.h>
+#include <linux/sched/task.h>
#include <linux/pagemap.h>
#include <linux/gfp.h>
#include <linux/swap.h>
*/
#include <linux/sched.h>
+#include <linux/sched/mm.h>
+#include <linux/sched/task.h>
#include <linux/mm.h>
#include <asm/tlbflush.h>
#ifndef _H8300_PGTABLE_H
#define _H8300_PGTABLE_H
+#define __ARCH_USE_5LEVEL_HACK
#include <asm-generic/pgtable-nopud.h>
#include <asm-generic/pgtable.h>
#define pgtable_cache_init() do { } while (0)
#include <linux/errno.h>
#include <linux/module.h>
#include <linux/sched.h>
+#include <linux/sched/debug.h>
+#include <linux/sched/task.h>
+#include <linux/sched/task_stack.h>
#include <linux/kernel.h>
#include <linux/mm.h>
#include <linux/smp.h>
*/
#include <linux/linkage.h>
-#include <linux/sched.h>
+#include <linux/sched/signal.h>
#include <asm/ptrace.h>
#define BREAKINST 0x5730 /* trapa #3 */
*/
#include <linux/linkage.h>
-#include <linux/sched.h>
+#include <linux/sched/signal.h>
#include <linux/errno.h>
#include <asm/ptrace.h>
*/
#include <linux/sched.h>
+#include <linux/sched/task_stack.h>
#include <linux/mm.h>
#include <linux/kernel.h>
#include <linux/signal.h>
#include <linux/types.h>
#include <linux/sched.h>
+#include <linux/sched/debug.h>
+#include <linux/mm_types.h>
#include <linux/kernel.h>
#include <linux/errno.h>
#include <linux/init.h>
#ifndef _ASM_MMU_CONTEXT_H
#define _ASM_MMU_CONTEXT_H
+#include <linux/mm_types.h>
+
#include <asm/setup.h>
#include <asm/page.h>
#include <asm/pgalloc.h>
*/
#include <linux/swap.h>
#include <asm/page.h>
+#define __ARCH_USE_5LEVEL_HACK
#include <asm-generic/pgtable-nopmd.h>
/* A handy thing to have if one has the RAM. Declared in head.S */
#include <linux/irq.h>
#include <linux/sched.h>
+#include <linux/sched/task_stack.h>
#include <linux/kdebug.h>
#include <linux/kgdb.h>
*/
#include <linux/sched.h>
+#include <linux/sched/debug.h>
+#include <linux/sched/task.h>
+#include <linux/sched/task_stack.h>
#include <linux/types.h>
#include <linux/module.h>
#include <linux/tick.h>
#include <linux/kernel.h>
#include <linux/sched.h>
+#include <linux/sched/task_stack.h>
#include <linux/mm.h>
#include <linux/smp.h>
#include <linux/errno.h>
#include <linux/linkage.h>
#include <linux/syscalls.h>
#include <linux/tracehook.h>
+#include <linux/sched/task_stack.h>
+
#include <asm/registers.h>
#include <asm/thread_info.h>
#include <asm/unistd.h>
#include <linux/interrupt.h>
#include <linux/module.h>
#include <linux/percpu.h>
-#include <linux/sched.h>
+#include <linux/sched/mm.h>
#include <linux/smp.h>
#include <linux/spinlock.h>
#include <linux/cpu.h>
+#include <linux/mm_types.h>
#include <asm/time.h> /* timer_interrupt */
#include <asm/hexagon_vm.h>
*/
#include <linux/sched.h>
+#include <linux/sched/task_stack.h>
#include <linux/stacktrace.h>
#include <linux/thread_info.h>
#include <linux/module.h>
*/
#include <linux/init.h>
-#include <linux/sched.h>
+#include <linux/sched/signal.h>
+#include <linux/sched/debug.h>
+#include <linux/sched/task_stack.h>
#include <linux/module.h>
#include <linux/kallsyms.h>
#include <linux/kdebug.h>
*/
#include <linux/kernel.h>
+#include <linux/sched/debug.h>
#include <asm/registers.h>
#include <linux/irq.h>
#include <linux/hardirq.h>
#include <asm/traps.h>
#include <linux/uaccess.h>
#include <linux/mm.h>
+#include <linux/sched/signal.h>
#include <linux/signal.h>
#include <linux/extable.h>
#include <linux/hardirq.h>
#include <linux/init.h>
#include <linux/errno.h>
#include <linux/sched.h>
+#include <linux/sched/debug.h>
#include <linux/tty.h>
#include <linux/tty_flip.h>
#include <linux/major.h>
#include <linux/compiler.h>
#include <linux/percpu.h>
#include <linux/sched.h>
+#include <linux/mm_types.h>
#include <linux/spinlock.h>
#include <asm/processor.h>
# ifndef __ASSEMBLY__
-#include <linux/sched.h> /* for mm_struct */
+#include <linux/sched/mm.h> /* for mm_struct */
#include <linux/bitops.h>
#include <asm/cacheflush.h>
#include <asm/mmu_context.h>
#if CONFIG_PGTABLE_LEVELS == 3
+#define __ARCH_USE_5LEVEL_HACK
#include <asm-generic/pgtable-nopud.h>
#endif
+#include <asm-generic/5level-fixup.h>
#include <asm-generic/pgtable.h>
#endif /* _ASM_IA64_PGTABLE_H */
#include <asm/ptrace.h>
#include <asm/ustack.h>
-#define ARCH_HAS_PREFETCH_SWITCH_STACK
-
#define IA64_NUM_PHYS_STACK_REG 96
#define IA64_NUM_DBG_REGS 8
#define ASM_OFFSETS_C 1
-#include <linux/sched.h>
+#include <linux/sched/signal.h>
#include <linux/pid.h>
#include <linux/clocksource.h>
#include <linux/kbuild.h>
*/
#include <linux/kernel.h>
-#include <linux/sched.h>
+#include <linux/sched/signal.h>
#include <linux/uaccess.h>
#include <asm/processor.h>
br.cond.sptk.many b7
END(load_switch_stack)
-GLOBAL_ENTRY(prefetch_stack)
- add r14 = -IA64_SWITCH_STACK_SIZE, sp
- add r15 = IA64_TASK_THREAD_KSP_OFFSET, in0
- ;;
- ld8 r16 = [r15] // load next's stack pointer
- lfetch.fault.excl [r14], 128
- ;;
- lfetch.fault.excl [r14], 128
- lfetch.fault [r16], 128
- ;;
- lfetch.fault.excl [r14], 128
- lfetch.fault [r16], 128
- ;;
- lfetch.fault.excl [r14], 128
- lfetch.fault [r16], 128
- ;;
- lfetch.fault.excl [r14], 128
- lfetch.fault [r16], 128
- ;;
- lfetch.fault [r16], 128
- br.ret.sptk.many rp
-END(prefetch_stack)
-
/*
* Invoke a system call, but do some tracing before and after the call.
* We MUST preserve the current register frame throughout this routine
#include <linux/jiffies.h>
#include <linux/types.h>
#include <linux/init.h>
-#include <linux/sched.h>
+#include <linux/sched/signal.h>
+#include <linux/sched/debug.h>
+#include <linux/sched/task.h>
#include <linux/interrupt.h>
#include <linux/irq.h>
#include <linux/bootmem.h>
#include <linux/module.h>
#include <linux/kernel.h>
#include <linux/sched.h>
+#include <linux/sched/task.h>
+#include <linux/sched/task_stack.h>
#include <linux/interrupt.h>
#include <linux/proc_fs.h>
#include <linux/seq_file.h>
#include <linux/notifier.h>
#include <linux/personality.h>
#include <linux/sched.h>
+#include <linux/sched/debug.h>
+#include <linux/sched/hotplug.h>
+#include <linux/sched/task.h>
+#include <linux/sched/task_stack.h>
#include <linux/stddef.h>
#include <linux/thread_info.h>
#include <linux/unistd.h>
*/
#include <linux/kernel.h>
#include <linux/sched.h>
+#include <linux/sched/task.h>
+#include <linux/sched/task_stack.h>
#include <linux/mm.h>
#include <linux/errno.h>
#include <linux/ptrace.h>
#include <linux/bootmem.h>
#include <linux/console.h>
#include <linux/delay.h>
+#include <linux/cpu.h>
#include <linux/kernel.h>
#include <linux/reboot.h>
-#include <linux/sched.h>
+#include <linux/sched/mm.h>
+#include <linux/sched/clock.h>
+#include <linux/sched/task_stack.h>
#include <linux/seq_file.h>
#include <linux/string.h>
#include <linux/threads.h>
#include <linux/mm.h>
#include <linux/mman.h>
#include <linux/sched.h>
+#include <linux/sched/mm.h>
+#include <linux/sched/task_stack.h>
#include <linux/shm.h>
#include <linux/file.h> /* doh, must come after sched.h... */
#include <linux/smp.h>
#include <linux/profile.h>
#include <linux/sched.h>
#include <linux/time.h>
+#include <linux/nmi.h>
#include <linux/interrupt.h>
#include <linux/efi.h>
#include <linux/timex.h>
#include <linux/timekeeper_internal.h>
#include <linux/platform_device.h>
-#include <linux/cputime.h>
+#include <linux/sched/cputime.h>
#include <asm/machvec.h>
#include <asm/delay.h>
#include <linux/kernel.h>
#include <linux/init.h>
-#include <linux/sched.h>
+#include <linux/sched/signal.h>
+#include <linux/sched/debug.h>
#include <linux/tty.h>
#include <linux/vt_kern.h> /* For unblank_screen() */
#include <linux/export.h>
*/
#include <linux/jiffies.h>
#include <linux/kernel.h>
-#include <linux/sched.h>
+#include <linux/sched/signal.h>
#include <linux/tty.h>
#include <linux/extable.h>
#include <linux/ratelimit.h>
#include <linux/errno.h>
#include <linux/string.h>
#include <linux/efi.h>
+#include <linux/nmi.h>
#include <linux/genalloc.h>
#include <linux/gfp.h>
#include <asm/page.h>
* Copyright (C) 1998-2002 Hewlett-Packard Co
* David Mosberger-Tang <davidm@hpl.hp.com>
*/
-#include <linux/sched.h>
+#include <linux/sched/signal.h>
#include <linux/kernel.h>
#include <linux/mm.h>
#include <linux/extable.h>
#include <linux/elf.h>
#include <linux/memblock.h>
#include <linux/mm.h>
+#include <linux/sched/signal.h>
#include <linux/mmzone.h>
#include <linux/module.h>
#include <linux/personality.h>
#include <linux/spinlock.h>
#include <linux/threads.h>
#include <linux/sched.h>
+#include <linux/mm_types.h>
#include <linux/smp.h>
#include <linux/interrupt.h>
#include <linux/irq.h>
#ifndef __ASSEMBLY__
#include <linux/atomic.h>
+#include <linux/mm_types.h>
+
#include <asm/pgalloc.h>
#include <asm/mmu.h>
#include <asm/tlbflush.h>
#include <linux/fs.h>
#include <linux/slab.h>
+#include <linux/sched/debug.h>
+#include <linux/sched/task.h>
+#include <linux/sched/task_stack.h>
#include <linux/module.h>
#include <linux/ptrace.h>
#include <linux/unistd.h>
#include <linux/kernel.h>
#include <linux/sched.h>
+#include <linux/sched/task_stack.h>
#include <linux/mm.h>
#include <linux/err.h>
#include <linux/smp.h>
#include <linux/kernel.h>
#include <linux/stddef.h>
#include <linux/fs.h>
-#include <linux/sched.h>
+#include <linux/sched/mm.h>
#include <linux/ioport.h>
#include <linux/mm.h>
#include <linux/bootmem.h>
#include <linux/kernel.h>
#include <linux/mm.h>
#include <linux/sched.h>
+#include <linux/sched/task.h>
#include <linux/err.h>
#include <linux/irq.h>
#include <linux/bootmem.h>
#include <linux/kallsyms.h>
#include <linux/stddef.h>
#include <linux/ptrace.h>
+#include <linux/sched/debug.h>
+#include <linux/sched/task_stack.h>
#include <linux/mm.h>
+#include <linux/cpu.h>
+
#include <asm/page.h>
#include <asm/processor.h>
#include <linux/user.h>
#include <linux/elfcore.h>
+#include <linux/mm_types.h>
/*
* fill in the user structure for an a.out core dump
#define __M68K_MMU_CONTEXT_H
#include <asm-generic/mm_hooks.h>
+#include <linux/mm_types.h>
static inline void enter_lazy_tlb(struct mm_struct *mm, struct task_struct *tsk)
{
#include <linux/errno.h>
#include <linux/module.h>
#include <linux/sched.h>
+#include <linux/sched/debug.h>
+#include <linux/sched/task.h>
+#include <linux/sched/task_stack.h>
#include <linux/kernel.h>
#include <linux/mm.h>
#include <linux/slab.h>
#include <linux/kernel.h>
#include <linux/sched.h>
+#include <linux/sched/task_stack.h>
#include <linux/mm.h>
#include <linux/smp.h>
#include <linux/errno.h>
#include <linux/export.h>
#include <linux/module.h>
#include <linux/sched.h>
+#include <linux/sched/loadavg.h>
#include <linux/kernel.h>
#include <linux/param.h>
#include <linux/string.h>
*/
#include <linux/sched.h>
+#include <linux/sched/debug.h>
#include <linux/signal.h>
#include <linux/kernel.h>
#include <linux/mm.h>
#include <linux/types.h>
#include <linux/kernel.h>
#include <linux/sched.h>
+#include <linux/sched/debug.h>
#include <linux/interrupt.h>
#include <linux/irq.h>
#include <linux/delay.h>
#include <linux/bootmem.h>
#include <linux/bitops.h>
#include <linux/module.h>
+#include <linux/sched/mm.h>
#include <asm/setup.h>
#include <asm/traps.h>
#include <asm/cacheflush.h>
#include <linux/io.h>
+#include <linux/mm_types.h>
static inline void enter_lazy_tlb(struct mm_struct *mm,
struct task_struct *tsk)
#define _METAG_PGTABLE_H
#include <asm/pgtable-bits.h>
+#define __ARCH_USE_5LEVEL_HACK
#include <asm-generic/pgtable-nopmd.h>
/* Invalid regions on Meta: 0x00000000-0x001FFFFF and 0xFFFF0000-0xFFFFFFFF */
#include <linux/errno.h>
#include <linux/export.h>
#include <linux/sched.h>
+#include <linux/sched/debug.h>
+#include <linux/sched/task.h>
+#include <linux/sched/task_stack.h>
#include <linux/kernel.h>
#include <linux/mm.h>
#include <linux/unistd.h>
#include <linux/tracehook.h>
#include <linux/elf.h>
#include <linux/uaccess.h>
+#include <linux/sched/task_stack.h>
+
#include <trace/syscall.h>
#define CREATE_TRACE_POINTS
*/
#include <linux/sched.h>
+#include <linux/sched/task_stack.h>
#include <linux/mm.h>
#include <linux/smp.h>
#include <linux/kernel.h>
#include <linux/delay.h>
#include <linux/init.h>
#include <linux/spinlock.h>
-#include <linux/sched.h>
+#include <linux/sched/mm.h>
+#include <linux/sched/hotplug.h>
+#include <linux/sched/task_stack.h>
#include <linux/interrupt.h>
#include <linux/cache.h>
#include <linux/profile.h>
#include <linux/export.h>
#include <linux/sched.h>
+#include <linux/sched/debug.h>
+#include <linux/sched/task_stack.h>
#include <linux/stacktrace.h>
#include <asm/stacktrace.h>
#include <linux/export.h>
#include <linux/sched.h>
+#include <linux/sched/debug.h>
+#include <linux/sched/task.h>
+#include <linux/sched/task_stack.h>
#include <linux/signal.h>
#include <linux/kernel.h>
#include <linux/mm.h>
#include <linux/mm.h>
#include <linux/kernel.h>
#include <linux/ptrace.h>
+#include <linux/sched/debug.h>
#include <linux/interrupt.h>
#include <linux/uaccess.h>
#include <linux/percpu.h>
#include <linux/memblock.h>
#include <linux/initrd.h>
+#include <linux/sched/task.h>
#include <asm/setup.h>
#include <asm/page.h>
#define _ASM_MICROBLAZE_MMU_CONTEXT_H
#include <linux/atomic.h>
+#include <linux/mm_types.h>
+
#include <asm/bitops.h>
#include <asm/mmu.h>
#include <asm-generic/mm_hooks.h>
# else /* CONFIG_MMU */
typedef struct { unsigned long ste[64]; } pmd_t;
typedef struct { pmd_t pue[1]; } pud_t;
-typedef struct { pud_t pge[1]; } pgd_t;
+typedef struct { pud_t p4e[1]; } p4d_t;
+typedef struct { p4d_t pge[1]; } pgd_t;
# endif /* CONFIG_MMU */
# define pte_val(x) ((x).pte)
#include <linux/kernel.h>
#include <linux/signal.h>
#include <linux/sched.h>
+#include <linux/sched/debug.h>
#include <linux/kallsyms.h>
#include <asm/exceptions.h>
*/
#include <linux/sched.h>
+#include <linux/sched/loadavg.h>
#include <linux/io.h>
#include <asm/setup.h>
#include <linux/cpu.h>
#include <linux/export.h>
#include <linux/sched.h>
+#include <linux/sched/debug.h>
+#include <linux/sched/task.h>
+#include <linux/sched/task_stack.h>
#include <linux/pm.h>
#include <linux/tick.h>
#include <linux/bitops.h>
#include <linux/kernel.h>
#include <linux/mm.h>
#include <linux/sched.h>
+#include <linux/sched/task_stack.h>
#include <linux/ptrace.h>
#include <linux/signal.h>
#include <linux/elf.h>
#include <linux/interrupt.h>
#include <linux/delay.h>
#include <linux/sched.h>
+#include <linux/sched/clock.h>
#include <linux/sched_clock.h>
#include <linux/clk.h>
#include <linux/clockchips.h>
#include <linux/kernel.h>
#include <linux/kallsyms.h>
#include <linux/sched.h>
+#include <linux/sched/debug.h>
#include <linux/debug_locks.h>
#include <asm/exceptions.h>
#include <linux/kallsyms.h>
#include <linux/kernel.h>
#include <linux/sched.h>
+#include <linux/sched/task_stack.h>
#include <linux/stacktrace.h>
#include <linux/types.h>
#include <linux/errno.h>
#include <linux/types.h>
#include <linux/vmalloc.h>
#include <linux/init.h>
+#include <linux/mm_types.h>
#include <asm/pgtable.h>
#include <asm/pgalloc.h>
#include <linux/irqflags.h>
#include <linux/notifier.h>
#include <linux/prefetch.h>
+#include <linux/ptrace.h>
#include <linux/sched.h>
+#include <linux/sched/task_stack.h>
#include <asm/cop2.h>
#include <asm/current.h>
#include <asm/cop2.h>
#include <linux/export.h>
#include <linux/interrupt.h>
+#include <linux/sched/task_stack.h>
#include "octeon-crypto.h"
#include <linux/interrupt.h>
#include <linux/kernel_stat.h>
#include <linux/sched.h>
+#include <linux/sched/hotplug.h>
+#include <linux/sched/task_stack.h>
#include <linux/init.h>
#include <linux/export.h>
#ifndef _ASM_ABI_H
#define _ASM_ABI_H
+#include <linux/signal_types.h>
+
#include <asm/signal.h>
#include <asm/siginfo.h>
#include <asm/vdso.h>
#include <linux/auxvec.h>
#include <linux/fs.h>
+#include <linux/mm_types.h>
+
#include <uapi/linux/elf.h>
#include <asm/current.h>
#define _ASM_FPU_H
#include <linux/sched.h>
+#include <linux/sched/task_stack.h>
+#include <linux/ptrace.h>
#include <linux/thread_info.h>
#include <linux/bitops.h>
#include <linux/errno.h>
#include <linux/sched.h>
+#include <linux/mm_types.h>
#include <linux/smp.h>
#include <linux/slab.h>
+
#include <asm/cacheflush.h>
#include <asm/dsemul.h>
#include <asm/hazards.h>
#include <asm/cachectl.h>
#include <asm/fixmap.h>
+#define __ARCH_USE_5LEVEL_HACK
#include <asm-generic/pgtable-nopmd.h>
extern int temp_tlb_entry;
#include <asm/cachectl.h>
#include <asm/fixmap.h>
+#define __ARCH_USE_5LEVEL_HACK
#if defined(CONFIG_PAGE_SIZE_64KB) && !defined(CONFIG_MIPS_VA_BITS_48)
#include <asm-generic/pgtable-nopmd.h>
#else
* Copyright (C) 2001 MIPS Technologies, Inc.
*/
#include <linux/kernel.h>
-#include <linux/sched.h>
+#include <linux/sched/signal.h>
#include <linux/signal.h>
#include <linux/export.h>
#include <asm/branch.h>
#include <linux/irq.h>
#include <linux/types.h>
#include <linux/sched.h>
+#include <linux/sched/task_stack.h>
/* This keeps a track of which one is crashing cpu. */
static int crashing_cpu = -1;
#include <linux/kernel.h>
#include <linux/init.h>
#include <linux/sched.h>
+#include <linux/sched/task.h>
+#include <linux/cred.h>
#include <linux/security.h>
#include <linux/types.h>
#include <linux/uaccess.h>
*/
#include <linux/perf_event.h>
+#include <linux/sched/task_stack.h>
#include <asm/stacktrace.h>
*/
#include <linux/errno.h>
#include <linux/sched.h>
+#include <linux/sched/debug.h>
+#include <linux/sched/task.h>
+#include <linux/sched/task_stack.h>
#include <linux/tick.h>
#include <linux/kernel.h>
#include <linux/mm.h>
#include <linux/elf.h>
#include <linux/kernel.h>
#include <linux/sched.h>
+#include <linux/sched/task_stack.h>
#include <linux/mm.h>
#include <linux/errno.h>
#include <linux/ptrace.h>
#include <linux/compat.h>
#include <linux/kernel.h>
#include <linux/sched.h>
+#include <linux/sched/task_stack.h>
#include <linux/mm.h>
#include <linux/errno.h>
#include <linux/ptrace.h>
#include <linux/syscalls.h>
#include <linux/moduleloader.h>
#include <linux/atomic.h>
+#include <linux/sched/signal.h>
+
#include <asm/mipsmtregs.h>
#include <asm/mips_mt.h>
#include <asm/processor.h>
#include <linux/compiler.h>
#include <linux/errno.h>
#include <linux/signal.h>
+#include <linux/sched/signal.h>
#include <linux/uaccess.h>
#include <asm/abi.h>
#include <linux/init.h>
#include <linux/sched.h>
+#include <linux/sched/hotplug.h>
+#include <linux/sched/task_stack.h>
#include <linux/mm.h>
#include <linux/delay.h>
#include <linux/smp.h>
#include <linux/delay.h>
#include <linux/io.h>
#include <linux/irqchip/mips-gic.h>
-#include <linux/sched.h>
+#include <linux/sched/task_stack.h>
+#include <linux/sched/hotplug.h>
#include <linux/slab.h>
#include <linux/smp.h>
#include <linux/types.h>
#include <linux/interrupt.h>
#include <linux/irqchip/mips-gic.h>
#include <linux/compiler.h>
+#include <linux/sched/task_stack.h>
#include <linux/smp.h>
#include <linux/atomic.h>
#include <linux/export.h>
#include <linux/time.h>
#include <linux/timex.h>
-#include <linux/sched.h>
+#include <linux/sched/mm.h>
#include <linux/cpumask.h>
#include <linux/cpu.h>
#include <linux/err.h>
* Copyright (C) 2006 Atsushi Nemoto <anemo@mba.ocn.ne.jp>
*/
#include <linux/sched.h>
+#include <linux/sched/debug.h>
+#include <linux/sched/task_stack.h>
#include <linux/stacktrace.h>
#include <linux/export.h>
#include <asm/stacktrace.h>
#include <linux/uaccess.h>
#include <linux/slab.h>
#include <linux/elf.h>
+#include <linux/sched/task_stack.h>
#include <asm/asm.h>
#include <asm/branch.h>
#include <linux/module.h>
#include <linux/extable.h>
#include <linux/mm.h>
-#include <linux/sched.h>
+#include <linux/sched/mm.h>
+#include <linux/sched/debug.h>
#include <linux/smp.h>
#include <linux/spinlock.h>
#include <linux/kallsyms.h>
#include <linux/module.h>
#include <linux/uaccess.h>
#include <linux/vmalloc.h>
+#include <linux/sched/signal.h>
#include <linux/fs.h>
#include <linux/bootmem.h>
+
#include <asm/fpu.h>
#include <asm/page.h>
#include <asm/cacheflush.h>
#include <linux/init.h>
#include <linux/sched.h>
#include <linux/notifier.h>
+#include <linux/ptrace.h>
#include <asm/fpu.h>
#include <asm/cop2.h>
#include <linux/init.h>
#include <linux/cpu.h>
#include <linux/sched.h>
+#include <linux/sched/hotplug.h>
+#include <linux/sched/task_stack.h>
#include <linux/smp.h>
#include <linux/cpufreq.h>
#include <asm/processor.h>
#include <linux/err.h>
#include <linux/slab.h>
+#include <linux/mm_types.h>
+#include <linux/sched/task.h>
#include <asm/branch.h>
#include <asm/cacheflush.h>
#include <linux/sched.h>
#include <linux/slab.h>
#include <linux/vmalloc.h>
+#include <linux/mm_types.h>
#include <asm/cacheflush.h>
#include <asm/io.h>
#include <asm/tlbflush.h>
#include <linux/export.h>
#include <linux/personality.h>
#include <linux/random.h>
-#include <linux/sched.h>
+#include <linux/sched/signal.h>
+#include <linux/sched/mm.h>
unsigned long shm_align_mask = PAGE_SIZE - 1; /* Sane caches */
EXPORT_SYMBOL(shm_align_mask);
#include <linux/kernel.h>
#include <linux/delay.h>
#include <linux/init.h>
+#include <linux/sched/task_stack.h>
#include <linux/smp.h>
#include <linux/irq.h>
* Copyright (C) 2009 Wind River Systems,
* written by Ralf Baechle <ralf@linux-mips.org>
*/
+#include <linux/capability.h>
#include <linux/init.h>
#include <linux/irqflags.h>
#include <linux/notifier.h>
#include <linux/prefetch.h>
+#include <linux/ptrace.h>
#include <linux/sched.h>
+#include <linux/sched/task_stack.h>
#include <asm/cop2.h>
#include <asm/current.h>
#include <linux/cpumask.h>
#include <linux/kernel.h>
#include <linux/sched.h>
+#include <linux/sched/task_stack.h>
#include <asm/mipsregs.h>
#include <asm/setup.h>
#include <linux/init.h>
#include <linux/kernel.h>
-#include <linux/sched.h>
+#include <linux/sched/signal.h>
#include <asm/addrspace.h>
#include <asm/traps.h>
#include <linux/rtc/ds1286.h>
#include <linux/interrupt.h>
#include <linux/kernel.h>
-#include <linux/sched.h>
+#include <linux/sched/signal.h>
#include <linux/notifier.h>
#include <linux/pm.h>
#include <linux/timer.h>
#include <linux/init.h>
#include <linux/kernel.h>
#include <linux/sched.h>
+#include <linux/sched/debug.h>
+#include <linux/sched/signal.h>
#include <linux/seq_file.h>
#include <asm/addrspace.h>
#include <linux/kernel.h>
#include <linux/signal.h> /* for SIGBUS */
#include <linux/sched.h> /* schow_regs(), force_sig() */
+#include <linux/sched/debug.h>
+#include <linux/sched/signal.h>
+#include <asm/ptrace.h>
#include <asm/sn/addrs.h>
#include <asm/sn/arch.h>
#include <asm/sn/sn0/hub.h>
*/
#include <linux/init.h>
#include <linux/sched.h>
+#include <linux/sched/task_stack.h>
+#include <linux/topology.h>
#include <linux/nodemask.h>
+
#include <asm/page.h>
#include <asm/processor.h>
+#include <asm/ptrace.h>
#include <asm/sn/arch.h>
#include <asm/sn/gda.h>
#include <asm/sn/intr.h>
#include <linux/init.h>
#include <linux/kernel.h>
#include <linux/sched.h>
+#include <linux/sched/debug.h>
+#include <linux/sched/signal.h>
#include <asm/traps.h>
#include <linux/uaccess.h>
#include <asm/addrspace.h>
#include <linux/mm.h>
#include <linux/random.h>
#include <linux/sched.h>
+#include <linux/sched/debug.h>
#include <asm/irq_cpu.h>
#include <asm/mipsregs.h>
#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/sched.h>
+#include <linux/sched/signal.h>
#include <linux/notifier.h>
#include <linux/delay.h>
#include <linux/rtc/ds1685.h>
#include <linux/smp.h>
#include <linux/kernel_stat.h>
#include <linux/sched.h>
+#include <linux/sched/task_stack.h>
#include <asm/mmu_context.h>
#include <asm/io.h>
#include <linux/interrupt.h>
#include <linux/smp.h>
#include <linux/kernel_stat.h>
-#include <linux/sched.h>
+#include <linux/sched/task_stack.h>
#include <asm/mmu_context.h>
#include <asm/io.h>
#define _ASM_MMU_CONTEXT_H
#include <linux/atomic.h>
+#include <linux/mm_types.h>
+
#include <asm/pgalloc.h>
#include <asm/tlbflush.h>
#include <asm-generic/mm_hooks.h>
#define __pgd(x) ((pgd_t) { (x) })
#define __pgprot(x) ((pgprot_t) { (x) })
+#define __ARCH_USE_5LEVEL_HACK
#include <asm-generic/pgtable-nopmd.h>
#endif /* !__ASSEMBLY__ */
* 2 of the Licence, or (at your option) any later version.
*/
#include <linux/uaccess.h>
+#include <linux/sched/signal.h>
+
#include <asm/fpu.h>
#include <asm/elf.h>
#include <asm/exceptions.h>
#include <linux/module.h>
#include <linux/errno.h>
#include <linux/sched.h>
+#include <linux/sched/debug.h>
+#include <linux/sched/task.h>
+#include <linux/sched/task_stack.h>
#include <linux/kernel.h>
#include <linux/mm.h>
#include <linux/smp.h>
*/
#include <linux/kernel.h>
#include <linux/sched.h>
+#include <linux/sched/task_stack.h>
#include <linux/mm.h>
#include <linux/smp.h>
#include <linux/errno.h>
#include <linux/err.h>
#include <linux/kernel.h>
#include <linux/delay.h>
-#include <linux/sched.h>
+#include <linux/sched/mm.h>
+#include <linux/sched/task.h>
#include <linux/profile.h>
#include <linux/smp.h>
#include <linux/cpu.h>
* 2 of the Licence, or (at your option) any later version.
*/
#include <linux/sched.h>
+#include <linux/sched/clock.h>
#include <linux/kernel.h>
#include <linux/interrupt.h>
#include <linux/time.h>
* 2 of the Licence, or (at your option) any later version.
*/
#include <linux/sched.h>
+#include <linux/sched/debug.h>
#include <linux/kernel.h>
#include <linux/string.h>
#include <linux/errno.h>
#include <linux/err.h>
#include <linux/kernel.h>
#include <linux/delay.h>
-#include <linux/sched.h>
+#include <linux/sched/mm.h>
#include <linux/profile.h>
#include <linux/smp.h>
#include <asm/tlbflush.h>
#ifndef _ASM_NIOS2_MMU_CONTEXT_H
#define _ASM_NIOS2_MMU_CONTEXT_H
+#include <linux/mm_types.h>
+
#include <asm-generic/mm_hooks.h>
extern void mmu_context_init(void);
#include <asm/tlbflush.h>
#include <asm/pgtable-bits.h>
+#define __ARCH_USE_5LEVEL_HACK
#include <asm-generic/pgtable-nopmd.h>
#define FIRST_USER_ADDRESS 0UL
#include <linux/export.h>
#include <linux/sched.h>
+#include <linux/sched/debug.h>
+#include <linux/sched/task.h>
+#include <linux/sched/task_stack.h>
+#include <linux/mm_types.h>
#include <linux/tick.h>
#include <linux/uaccess.h>
#include <linux/ptrace.h>
#include <linux/regset.h>
#include <linux/sched.h>
+#include <linux/sched/task_stack.h>
#include <linux/tracehook.h>
#include <linux/uaccess.h>
#include <linux/user.h>
#include <linux/kernel.h>
#include <linux/mm.h>
#include <linux/sched.h>
+#include <linux/sched/task.h>
#include <linux/console.h>
#include <linux/bootmem.h>
#include <linux/initrd.h>
*/
#include <linux/sched.h>
+#include <linux/sched/debug.h>
#include <linux/kernel.h>
#include <linux/signal.h>
#include <linux/export.h>
#include <linux/signal.h>
#include <linux/sched.h>
+#include <linux/sched/debug.h>
#include <linux/interrupt.h>
#include <linux/kernel.h>
#include <linux/errno.h>
#ifndef __ASM_OPENRISC_PGTABLE_H
#define __ASM_OPENRISC_PGTABLE_H
+#define __ARCH_USE_5LEVEL_HACK
#include <asm-generic/pgtable-nopmd.h>
#ifndef __ASSEMBLY__
#include <linux/errno.h>
#include <linux/sched.h>
+#include <linux/sched/debug.h>
+#include <linux/sched/task.h>
+#include <linux/sched/task_stack.h>
#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/mm.h>
#include <linux/kernel.h>
#include <linux/sched.h>
+#include <linux/sched/task_stack.h>
#include <linux/string.h>
#include <linux/mm.h>
#include <linux/init.h>
#include <linux/sched.h>
+#include <linux/sched/debug.h>
+#include <linux/sched/task_stack.h>
#include <linux/kernel.h>
#include <linux/extable.h>
#include <linux/kmod.h>
#include <linux/mm.h>
#include <linux/interrupt.h>
#include <linux/extable.h>
-#include <linux/sched.h>
+#include <linux/sched/signal.h>
#include <linux/uaccess.h>
#include <asm/siginfo.h>
void flush_kernel_dcache_range_asm(unsigned long, unsigned long);
void flush_kernel_dcache_page_asm(void *);
void flush_kernel_icache_page(void *);
-void flush_user_dcache_range(unsigned long, unsigned long);
-void flush_user_icache_range(unsigned long, unsigned long);
/* Cache flush operations */
* that put_user is the same as __put_user, etc.
*/
-static inline long access_ok(int type, const void __user * addr,
- unsigned long size)
-{
- return 1;
-}
+#define access_ok(type, uaddr, size) (1)
#define put_user __put_user
#define get_user __get_user
#include <linux/seq_file.h>
#include <linux/pagemap.h>
#include <linux/sched.h>
+#include <linux/sched/mm.h>
#include <asm/pdc.h>
#include <asm/cache.h>
#include <asm/cacheflush.h>
}
}
-void
-flush_user_dcache_range(unsigned long start, unsigned long end)
-{
- if ((end - start) < parisc_cache_flush_threshold)
- flush_user_dcache_range_asm(start,end);
- else
- flush_data_cache();
-}
-
-void
-flush_user_icache_range(unsigned long start, unsigned long end)
-{
- if ((end - start) < parisc_cache_flush_threshold)
- flush_user_icache_range_asm(start,end);
- else
- flush_instruction_cache();
-}
-
void flush_cache_range(struct vm_area_struct *vma,
unsigned long start, unsigned long end)
{
* Copyright (C) 2000 Philipp Rumpf */
#include <linux/sched.h>
+#include <linux/sched/debug.h>
#include <linux/smp.h>
#include <linux/kernel.h>
#include <asm/io.h>
#include <linux/personality.h>
#include <linux/ptrace.h>
#include <linux/sched.h>
+#include <linux/sched/debug.h>
+#include <linux/sched/task.h>
+#include <linux/sched/task_stack.h>
#include <linux/slab.h>
#include <linux/stddef.h>
#include <linux/unistd.h>
#include <linux/proc_fs.h>
#include <linux/export.h>
#include <linux/sched.h>
+#include <linux/sched/clock.h>
#include <asm/processor.h>
#include <asm/sections.h>
*/
#include <linux/sched.h>
+#include <linux/sched/debug.h>
#include <linux/mm.h>
#include <linux/smp.h>
#include <linux/kernel.h>
struct rt_sigframe __user *frame;
unsigned long rp, usp;
unsigned long haddr, sigframe_size;
+ unsigned long start, end;
int err = 0;
#ifdef CONFIG_64BIT
struct compat_rt_sigframe __user * compat_frame;
}
#endif
- flush_user_dcache_range((unsigned long) &frame->tramp[0],
- (unsigned long) &frame->tramp[TRAMP_SIZE]);
- flush_user_icache_range((unsigned long) &frame->tramp[0],
- (unsigned long) &frame->tramp[TRAMP_SIZE]);
+ start = (unsigned long) &frame->tramp[0];
+ end = (unsigned long) &frame->tramp[TRAMP_SIZE];
+ flush_user_dcache_range_asm(start, end);
+ flush_user_icache_range_asm(start, end);
/* TRAMP Words 0-4, Length 5 = SIGRESTARTBLOCK_TRAMP
* TRAMP Words 5-9, Length 4 = SIGRETURN_TRAMP
WARN_ON(err);
/* flush data/instruction cache for new insns */
- flush_user_dcache_range(start, end);
- flush_user_icache_range(start, end);
+ flush_user_dcache_range_asm(start, end);
+ flush_user_icache_range_asm(start, end);
regs->gr[31] = regs->gr[30] + 8;
return;
#include <linux/kernel.h>
#include <linux/module.h>
-#include <linux/sched.h>
+#include <linux/sched/mm.h>
#include <linux/init.h>
#include <linux/interrupt.h>
#include <linux/smp.h>
#include <linux/linkage.h>
#include <linux/mm.h>
#include <linux/mman.h>
+#include <linux/sched/signal.h>
+#include <linux/sched/mm.h>
#include <linux/shm.h>
#include <linux/syscalls.h>
#include <linux/utsname.h>
#include <linux/module.h>
#include <linux/rtc.h>
#include <linux/sched.h>
+#include <linux/sched/clock.h>
#include <linux/sched_clock.h>
#include <linux/kernel.h>
#include <linux/param.h>
*/
#include <linux/sched.h>
+#include <linux/sched/debug.h>
#include <linux/kernel.h>
#include <linux/string.h>
#include <linux/errno.h>
#include <linux/jiffies.h>
#include <linux/kernel.h>
#include <linux/module.h>
-#include <linux/sched.h>
+#include <linux/sched/signal.h>
+#include <linux/sched/debug.h>
#include <linux/signal.h>
#include <linux/ratelimit.h>
#include <linux/uaccess.h>
* Copyright (C) 2001 Hewlett-Packard <bame@debian.org>
*/
-#include <linux/sched.h>
+#include <linux/sched/signal.h>
+
#include "float.h"
#include "math-emu.h"
#include <linux/mm.h>
#include <linux/ptrace.h>
#include <linux/sched.h>
+#include <linux/sched/debug.h>
#include <linux/interrupt.h>
#include <linux/extable.h>
#include <linux/uaccess.h>
vma ? ',':'\n');
if (vma)
- pr_warn(KERN_CONT " vm_start = 0x%08lx, vm_end = 0x%08lx\n",
- vma->vm_start, vma->vm_end);
+ pr_cont(" vm_start = 0x%08lx, vm_end = 0x%08lx\n",
+ vma->vm_start, vma->vm_end);
show_regs(regs);
}
#include <linux/fs.h>
#include <linux/mm.h>
+#include <linux/sched/mm.h>
#include <linux/hugetlb.h>
#include <linux/pagemap.h>
#include <linux/sysctl.h>
config PPC
bool
default y
- select BUILDTIME_EXTABLE_SORT
+ #
+ # Please keep this list sorted alphabetically.
+ #
+ select ARCH_HAS_DEVMEM_IS_ALLOWED
+ select ARCH_HAS_DMA_SET_COHERENT_MASK
+ select ARCH_HAS_ELF_RANDOMIZE
+ select ARCH_HAS_GCOV_PROFILE_ALL
+ select ARCH_HAS_SCALED_CPUTIME if VIRT_CPU_ACCOUNTING_NATIVE
+ select ARCH_HAS_SG_CHAIN
+ select ARCH_HAS_TICK_BROADCAST if GENERIC_CLOCKEVENTS_BROADCAST
+ select ARCH_HAS_UBSAN_SANITIZE_ALL
+ select ARCH_HAVE_NMI_SAFE_CMPXCHG
select ARCH_MIGHT_HAVE_PC_PARPORT
select ARCH_MIGHT_HAVE_PC_SERIO
+ select ARCH_SUPPORTS_ATOMIC_RMW
+ select ARCH_SUPPORTS_DEFERRED_STRUCT_PAGE_INIT
+ select ARCH_USE_BUILTIN_BSWAP
+ select ARCH_USE_CMPXCHG_LOCKREF if PPC64
+ select ARCH_WANT_IPC_PARSE_VERSION
select BINFMT_ELF
- select ARCH_HAS_ELF_RANDOMIZE
- select OF
- select OF_EARLY_FLATTREE
- select OF_RESERVED_MEM
- select HAVE_FTRACE_MCOUNT_RECORD
+ select BUILDTIME_EXTABLE_SORT
+ select CLONE_BACKWARDS
+ select DCACHE_WORD_ACCESS if PPC64 && CPU_LITTLE_ENDIAN
+ select EDAC_ATOMIC_SCRUB
+ select EDAC_SUPPORT
+ select GENERIC_ATOMIC64 if PPC32
+ select GENERIC_CLOCKEVENTS
+ select GENERIC_CLOCKEVENTS_BROADCAST if SMP
+ select GENERIC_CMOS_UPDATE
+ select GENERIC_CPU_AUTOPROBE
+ select GENERIC_IRQ_SHOW
+ select GENERIC_IRQ_SHOW_LEVEL
+ select GENERIC_SMP_IDLE_THREAD
+ select GENERIC_STRNCPY_FROM_USER
+ select GENERIC_STRNLEN_USER
+ select GENERIC_TIME_VSYSCALL_OLD
+ select HAVE_ARCH_AUDITSYSCALL
+ select HAVE_ARCH_HARDENED_USERCOPY
+ select HAVE_ARCH_JUMP_LABEL
+ select HAVE_ARCH_KGDB
+ select HAVE_ARCH_SECCOMP_FILTER
+ select HAVE_ARCH_TRACEHOOK
+ select HAVE_CBPF_JIT if !PPC64
+ select HAVE_CONTEXT_TRACKING if PPC64
+ select HAVE_DEBUG_KMEMLEAK
+ select HAVE_DEBUG_STACKOVERFLOW
+ select HAVE_DMA_API_DEBUG
select HAVE_DYNAMIC_FTRACE
- select HAVE_DYNAMIC_FTRACE_WITH_REGS if MPROFILE_KERNEL
- select HAVE_FUNCTION_TRACER
+ select HAVE_DYNAMIC_FTRACE_WITH_REGS if MPROFILE_KERNEL
+ select HAVE_EBPF_JIT if PPC64
+ select HAVE_EFFICIENT_UNALIGNED_ACCESS if !(CPU_LITTLE_ENDIAN && POWER7_CPU)
+ select HAVE_FTRACE_MCOUNT_RECORD
select HAVE_FUNCTION_GRAPH_TRACER
+ select HAVE_FUNCTION_TRACER
select HAVE_GCC_PLUGINS
- select SYSCTL_EXCEPTION_TRACE
- select VIRT_TO_BUS if !PPC64
+ select HAVE_GENERIC_RCU_GUP
+ select HAVE_HW_BREAKPOINT if PERF_EVENTS && (PPC_BOOK3S || PPC_8xx)
select HAVE_IDE
select HAVE_IOREMAP_PROT
- select HAVE_EFFICIENT_UNALIGNED_ACCESS if !(CPU_LITTLE_ENDIAN && POWER7_CPU)
+ select HAVE_IRQ_EXIT_ON_IRQ_STACK
+ select HAVE_KERNEL_GZIP
select HAVE_KPROBES
- select HAVE_OPTPROBES if PPC64
- select HAVE_ARCH_KGDB
select HAVE_KRETPROBES
- select HAVE_ARCH_TRACEHOOK
+ select HAVE_LIVEPATCH if HAVE_DYNAMIC_FTRACE_WITH_REGS
select HAVE_MEMBLOCK
select HAVE_MEMBLOCK_NODE_MAP
- select HAVE_DMA_API_DEBUG
+ select HAVE_MOD_ARCH_SPECIFIC
+ select HAVE_NMI if PERF_EVENTS
select HAVE_OPROFILE
- select HAVE_DEBUG_KMEMLEAK
- select ARCH_HAS_SG_CHAIN
- select GENERIC_ATOMIC64 if PPC32
+ select HAVE_OPTPROBES if PPC64
select HAVE_PERF_EVENTS
+ select HAVE_PERF_EVENTS_NMI if PPC64
select HAVE_PERF_REGS
select HAVE_PERF_USER_STACK_DUMP
+ select HAVE_RCU_TABLE_FREE if SMP
select HAVE_REGS_AND_STACK_ACCESS_API
- select HAVE_HW_BREAKPOINT if PERF_EVENTS && (PPC_BOOK3S || PPC_8xx)
- select ARCH_WANT_IPC_PARSE_VERSION
- select SPARSE_IRQ
+ select HAVE_SYSCALL_TRACEPOINTS
+ select HAVE_VIRT_CPU_ACCOUNTING
select IRQ_DOMAIN
- select GENERIC_IRQ_SHOW
- select GENERIC_IRQ_SHOW_LEVEL
select IRQ_FORCED_THREADING
- select HAVE_RCU_TABLE_FREE if SMP
- select HAVE_SYSCALL_TRACEPOINTS
- select HAVE_CBPF_JIT if !PPC64
- select HAVE_EBPF_JIT if PPC64
- select HAVE_ARCH_JUMP_LABEL
- select ARCH_HAVE_NMI_SAFE_CMPXCHG
- select ARCH_HAS_GCOV_PROFILE_ALL
- select GENERIC_SMP_IDLE_THREAD
- select GENERIC_CMOS_UPDATE
- select GENERIC_TIME_VSYSCALL_OLD
- select GENERIC_CLOCKEVENTS
- select GENERIC_CLOCKEVENTS_BROADCAST if SMP
- select ARCH_HAS_TICK_BROADCAST if GENERIC_CLOCKEVENTS_BROADCAST
- select GENERIC_STRNCPY_FROM_USER
- select GENERIC_STRNLEN_USER
- select HAVE_MOD_ARCH_SPECIFIC
select MODULES_USE_ELF_RELA
- select CLONE_BACKWARDS
- select ARCH_USE_BUILTIN_BSWAP
- select OLD_SIGSUSPEND
- select OLD_SIGACTION if PPC32
- select HAVE_DEBUG_STACKOVERFLOW
- select HAVE_IRQ_EXIT_ON_IRQ_STACK
- select ARCH_USE_CMPXCHG_LOCKREF if PPC64
- select HAVE_ARCH_AUDITSYSCALL
- select ARCH_SUPPORTS_ATOMIC_RMW
- select DCACHE_WORD_ACCESS if PPC64 && CPU_LITTLE_ENDIAN
select NO_BOOTMEM
- select HAVE_GENERIC_RCU_GUP
- select HAVE_PERF_EVENTS_NMI if PPC64
- select HAVE_NMI if PERF_EVENTS
- select EDAC_SUPPORT
- select EDAC_ATOMIC_SCRUB
- select ARCH_HAS_DMA_SET_COHERENT_MASK
- select ARCH_HAS_DEVMEM_IS_ALLOWED
- select HAVE_ARCH_SECCOMP_FILTER
- select ARCH_HAS_UBSAN_SANITIZE_ALL
- select ARCH_SUPPORTS_DEFERRED_STRUCT_PAGE_INIT
- select HAVE_LIVEPATCH if HAVE_DYNAMIC_FTRACE_WITH_REGS
- select GENERIC_CPU_AUTOPROBE
- select HAVE_VIRT_CPU_ACCOUNTING
- select ARCH_HAS_SCALED_CPUTIME if VIRT_CPU_ACCOUNTING_NATIVE
- select HAVE_ARCH_HARDENED_USERCOPY
- select HAVE_KERNEL_GZIP
- select HAVE_CONTEXT_TRACKING if PPC64
+ select OF
+ select OF_EARLY_FLATTREE
+ select OF_RESERVED_MEM
+ select OLD_SIGACTION if PPC32
+ select OLD_SIGSUSPEND
+ select SPARSE_IRQ
+ select SYSCTL_EXCEPTION_TRACE
+ select VIRT_TO_BUS if !PPC64
+ #
+ # Please keep this list sorted alphabetically.
+ #
config GENERIC_CSUM
def_bool n
MULTIPLEWORD := -mmultiple
endif
-cflags-$(CONFIG_CPU_BIG_ENDIAN) += $(call cc-option,-mbig-endian)
+ifdef CONFIG_PPC64
+cflags-$(CONFIG_CPU_BIG_ENDIAN) += $(call cc-option,-mabi=elfv1)
+cflags-$(CONFIG_CPU_BIG_ENDIAN) += $(call cc-option,-mcall-aixdesc)
+aflags-$(CONFIG_CPU_BIG_ENDIAN) += $(call cc-option,-mabi=elfv1)
+aflags-$(CONFIG_CPU_LITTLE_ENDIAN) += -mabi=elfv2
+endif
+
cflags-$(CONFIG_CPU_LITTLE_ENDIAN) += -mlittle-endian
+cflags-$(CONFIG_CPU_BIG_ENDIAN) += $(call cc-option,-mbig-endian)
ifneq ($(cc-name),clang)
cflags-$(CONFIG_CPU_LITTLE_ENDIAN) += -mno-strict-align
endif
CFLAGS-$(CONFIG_PPC64) += $(call cc-option,-mabi=elfv2,$(call cc-option,-mcall-aixdesc))
AFLAGS-$(CONFIG_PPC64) += $(call cc-option,-mabi=elfv2)
else
+CFLAGS-$(CONFIG_PPC64) += $(call cc-option,-mabi=elfv1)
CFLAGS-$(CONFIG_PPC64) += $(call cc-option,-mcall-aixdesc)
+AFLAGS-$(CONFIG_PPC64) += $(call cc-option,-mabi=elfv1)
endif
CFLAGS-$(CONFIG_PPC64) += $(call cc-option,-mcmodel=medium,$(call cc-option,-mminimal-toc))
CFLAGS-$(CONFIG_PPC64) += $(call cc-option,-mno-pointers-to-nested-functions)
#ifndef _ASM_POWERPC_BOOK3S_32_PGTABLE_H
#define _ASM_POWERPC_BOOK3S_32_PGTABLE_H
+#define __ARCH_USE_5LEVEL_HACK
#include <asm-generic/pgtable-nopmd.h>
#include <asm/book3s/32/hash.h>
/* Bits in patb0 field */
#define PATB_HR (1UL << 63)
-#define RPDB_MASK 0x0ffffffffffff00fUL
+#define RPDB_MASK 0x0fffffffffffff00UL
#define RPDB_SHIFT (1UL << 8)
#define RTS1_SHIFT 61 /* top 2 bits of radix tree size */
#define RTS1_MASK (3UL << RTS1_SHIFT)
/* Bits in patb1 field */
#define PATB_GR (1UL << 63) /* guest uses radix; must match HR */
#define PRTS_MASK 0x1f /* process table size field */
+#define PRTB_MASK 0x0ffffffffffff000UL
/*
* Limit process table to PAGE_SIZE table. This
#ifndef _ASM_POWERPC_BOOK3S_64_PGTABLE_H_
#define _ASM_POWERPC_BOOK3S_64_PGTABLE_H_
+#include <asm-generic/5level-fixup.h>
+
#ifndef __ASSEMBLY__
#include <linux/mmdebug.h>
#endif
+
/*
* Common bits between hash and Radix page table
*/
__r; \
})
+static inline int __pte_write(pte_t pte)
+{
+ return !!(pte_raw(pte) & cpu_to_be64(_PAGE_WRITE));
+}
+
+#ifdef CONFIG_NUMA_BALANCING
+#define pte_savedwrite pte_savedwrite
+static inline bool pte_savedwrite(pte_t pte)
+{
+ /*
+ * Saved write ptes are prot none ptes that doesn't have
+ * privileged bit sit. We mark prot none as one which has
+ * present and pviliged bit set and RWX cleared. To mark
+ * protnone which used to have _PAGE_WRITE set we clear
+ * the privileged bit.
+ */
+ return !(pte_raw(pte) & cpu_to_be64(_PAGE_RWX | _PAGE_PRIVILEGED));
+}
+#else
+#define pte_savedwrite pte_savedwrite
+static inline bool pte_savedwrite(pte_t pte)
+{
+ return false;
+}
+#endif
+
+static inline int pte_write(pte_t pte)
+{
+ return __pte_write(pte) || pte_savedwrite(pte);
+}
+
#define __HAVE_ARCH_PTEP_SET_WRPROTECT
static inline void ptep_set_wrprotect(struct mm_struct *mm, unsigned long addr,
pte_t *ptep)
{
- if ((pte_raw(*ptep) & cpu_to_be64(_PAGE_WRITE)) == 0)
- return;
-
- pte_update(mm, addr, ptep, _PAGE_WRITE, 0, 0);
+ if (__pte_write(*ptep))
+ pte_update(mm, addr, ptep, _PAGE_WRITE, 0, 0);
+ else if (unlikely(pte_savedwrite(*ptep)))
+ pte_update(mm, addr, ptep, 0, _PAGE_PRIVILEGED, 0);
}
static inline void huge_ptep_set_wrprotect(struct mm_struct *mm,
unsigned long addr, pte_t *ptep)
{
- if ((pte_raw(*ptep) & cpu_to_be64(_PAGE_WRITE)) == 0)
- return;
-
- pte_update(mm, addr, ptep, _PAGE_WRITE, 0, 1);
+ /*
+ * We should not find protnone for hugetlb, but this complete the
+ * interface.
+ */
+ if (__pte_write(*ptep))
+ pte_update(mm, addr, ptep, _PAGE_WRITE, 0, 1);
+ else if (unlikely(pte_savedwrite(*ptep)))
+ pte_update(mm, addr, ptep, 0, _PAGE_PRIVILEGED, 1);
}
#define __HAVE_ARCH_PTEP_GET_AND_CLEAR
pte_update(mm, addr, ptep, ~0UL, 0, 0);
}
-static inline int pte_write(pte_t pte)
-{
- return !!(pte_raw(pte) & cpu_to_be64(_PAGE_WRITE));
-}
-
static inline int pte_dirty(pte_t pte)
{
return !!(pte_raw(pte) & cpu_to_be64(_PAGE_DIRTY));
VM_BUG_ON(!pte_protnone(pte));
return __pte(pte_val(pte) | _PAGE_PRIVILEGED);
}
-
-#define pte_savedwrite pte_savedwrite
-static inline bool pte_savedwrite(pte_t pte)
+#else
+#define pte_clear_savedwrite pte_clear_savedwrite
+static inline pte_t pte_clear_savedwrite(pte_t pte)
{
- /*
- * Saved write ptes are prot none ptes that doesn't have
- * privileged bit sit. We mark prot none as one which has
- * present and pviliged bit set and RWX cleared. To mark
- * protnone which used to have _PAGE_WRITE set we clear
- * the privileged bit.
- */
- VM_BUG_ON(!pte_protnone(pte));
- return !(pte_raw(pte) & cpu_to_be64(_PAGE_RWX | _PAGE_PRIVILEGED));
+ VM_WARN_ON(1);
+ return __pte(pte_val(pte) & ~_PAGE_WRITE);
}
#endif /* CONFIG_NUMA_BALANCING */
/* Generic modifiers for PTE bits */
static inline pte_t pte_wrprotect(pte_t pte)
{
+ if (unlikely(pte_savedwrite(pte)))
+ return pte_clear_savedwrite(pte);
return __pte(pte_val(pte) & ~_PAGE_WRITE);
}
#define __HAVE_ARCH_PMD_WRITE
#define pmd_write(pmd) pte_write(pmd_pte(pmd))
+#define __pmd_write(pmd) __pte_write(pmd_pte(pmd))
#define pmd_savedwrite(pmd) pte_savedwrite(pmd_pte(pmd))
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
static inline void pmdp_set_wrprotect(struct mm_struct *mm, unsigned long addr,
pmd_t *pmdp)
{
-
- if ((pmd_raw(*pmdp) & cpu_to_be64(_PAGE_WRITE)) == 0)
- return;
-
- pmd_hugepage_update(mm, addr, pmdp, _PAGE_WRITE, 0);
+ if (__pmd_write((*pmdp)))
+ pmd_hugepage_update(mm, addr, pmdp, _PAGE_WRITE, 0);
+ else if (unlikely(pmd_savedwrite(*pmdp)))
+ pmd_hugepage_update(mm, addr, pmdp, 0, _PAGE_PRIVILEGED);
}
static inline int pmd_trans_huge(pmd_t pmd)
#ifdef __powerpc64__
res += (__force u64)addend;
- return (__force __wsum)((u32)res + (res >> 32));
+ return (__force __wsum) from64to32(res);
#else
asm("addc %0,%0,%1;"
"addze %0,%0;"
std r0,0(r1); \
ptesync; \
ld r0,0(r1); \
-1: cmpd cr0,r0,r0; \
- bne 1b; \
+236: cmpd cr0,r0,r0; \
+ bne 236b; \
IDLE_INST; \
#define IDLE_STATE_ENTER_SEQ_NORET(IDLE_INST) \
#define ARCH_DLINFO_CACHE_GEOMETRY \
NEW_AUX_ENT(AT_L1I_CACHESIZE, ppc64_caches.l1i.size); \
NEW_AUX_ENT(AT_L1I_CACHEGEOMETRY, get_cache_geometry(l1i)); \
- NEW_AUX_ENT(AT_L1D_CACHESIZE, ppc64_caches.l1i.size); \
- NEW_AUX_ENT(AT_L1D_CACHEGEOMETRY, get_cache_geometry(l1i)); \
+ NEW_AUX_ENT(AT_L1D_CACHESIZE, ppc64_caches.l1d.size); \
+ NEW_AUX_ENT(AT_L1D_CACHEGEOMETRY, get_cache_geometry(l1d)); \
NEW_AUX_ENT(AT_L2_CACHESIZE, ppc64_caches.l2.size); \
NEW_AUX_ENT(AT_L2_CACHEGEOMETRY, get_cache_geometry(l2)); \
NEW_AUX_ENT(AT_L3_CACHESIZE, ppc64_caches.l3.size); \
#ifndef _ASM_POWERPC_NOHASH_32_PGTABLE_H
#define _ASM_POWERPC_NOHASH_32_PGTABLE_H
+#define __ARCH_USE_5LEVEL_HACK
#include <asm-generic/pgtable-nopmd.h>
#ifndef __ASSEMBLY__
#ifndef _ASM_POWERPC_NOHASH_64_PGTABLE_4K_H
#define _ASM_POWERPC_NOHASH_64_PGTABLE_4K_H
+
+#include <asm-generic/5level-fixup.h>
+
/*
* Entries per page directory level. The PTE level must use a 64b record
* for each page table entry. The PMD and PGD level use a 32b record for
#ifndef _ASM_POWERPC_NOHASH_64_PGTABLE_64K_H
#define _ASM_POWERPC_NOHASH_64_PGTABLE_64K_H
+#define __ARCH_USE_5LEVEL_HACK
#include <asm-generic/pgtable-nopud.h>
return ((hpd_val(hpd) & 0x4) != 0);
#else
/* We clear the top bit to indicate hugepd */
- return ((hpd_val(hpd) & PD_HUGE) == 0);
+ return (hpd_val(hpd) && (hpd_val(hpd) & PD_HUGE) == 0);
#endif
}
#define PPC_INST_BRANCH_COND 0x40800000
#define PPC_INST_LBZCIX 0x7c0006aa
#define PPC_INST_STBCIX 0x7c0007aa
+#define PPC_INST_LWZX 0x7c00002e
+#define PPC_INST_LFSX 0x7c00042e
+#define PPC_INST_STFSX 0x7c00052e
+#define PPC_INST_LFDX 0x7c0004ae
+#define PPC_INST_STFDX 0x7c0005ae
+#define PPC_INST_LVX 0x7c0000ce
+#define PPC_INST_STVX 0x7c0001ce
/* macros to insert fields into opcodes */
#define ___PPC_RA(a) (((a) & 0x1f) << 16)
#define OV5_PFO_HW_ENCR 0x1120 /* PFO Encryption Accelerator */
#define OV5_SUB_PROCESSORS 0x1501 /* 1,2,or 4 Sub-Processors supported */
#define OV5_XIVE_EXPLOIT 0x1701 /* XIVE exploitation supported */
-#define OV5_MMU_RADIX_300 0x1880 /* ISA v3.00 radix MMU supported */
-#define OV5_MMU_HASH_300 0x1840 /* ISA v3.00 hash MMU supported */
-#define OV5_MMU_SEGM_RADIX 0x1820 /* radix mode (no segmentation) */
-#define OV5_MMU_PROC_TBL 0x1810 /* hcall selects SLB or proc table */
-#define OV5_MMU_SLB 0x1800 /* always use SLB */
-#define OV5_MMU_GTSE 0x1808 /* Guest translation shootdown */
+/* MMU Base Architecture */
+#define OV5_MMU_SUPPORT 0x18C0 /* MMU Mode Support Mask */
+#define OV5_MMU_HASH 0x1800 /* Hash MMU Only */
+#define OV5_MMU_RADIX 0x1840 /* Radix MMU Only */
+#define OV5_MMU_EITHER 0x1880 /* Hash or Radix Supported */
+#define OV5_MMU_DYNAMIC 0x18C0 /* Hash or Radix Can Switch Later */
+#define OV5_NMMU 0x1820 /* Nest MMU Available */
+/* Hash Table Extensions */
+#define OV5_HASH_SEG_TBL 0x1980 /* In Memory Segment Tables Available */
+#define OV5_HASH_GTSE 0x1940 /* Guest Translation Shoot Down Avail */
+/* Radix Table Extensions */
+#define OV5_RADIX_GTSE 0x1A40 /* Guest Translation Shoot Down Avail */
/* Option Vector 6: IBM PAPR hints */
#define OV6_LINUX 0x02 /* Linux is our OS */
*/
andis. r4,r3,PSSCR_EC_ESL_MASK_SHIFTED
clrldi r3,r3,60 /* r3 = Bits[60:63] = Requested Level (RL) */
- bne 1f
+ bne .Lhandle_esl_ec_set
IDLE_STATE_ENTER_SEQ(PPC_STOP)
li r3,0 /* Since we didn't lose state, return 0 */
b pnv_wakeup_noloss
+
+.Lhandle_esl_ec_set:
/*
* Check if the requested state is a deep idle state.
*/
-1: LOAD_REG_ADDRBASE(r5,pnv_first_deep_stop_state)
+ LOAD_REG_ADDRBASE(r5,pnv_first_deep_stop_state)
ld r4,ADDROFF(pnv_first_deep_stop_state)(r5)
cmpd r3,r4
- bge 2f
+ bge .Lhandle_deep_stop
IDLE_STATE_ENTER_SEQ_NORET(PPC_STOP)
-2:
+.Lhandle_deep_stop:
/*
* Entering deep idle state.
* Clear thread bit in PACA_CORE_IDLE_STATE, save SPRs to
#undef DEBUG
#include <linux/kernel.h>
-#include <linux/sched.h> /* for init_mm */
+#include <linux/sched/mm.h> /* for init_mm */
#include <asm/io.h>
#include <asm/machdep.h>
#include <linux/smp.h>
#include <linux/export.h>
#include <linux/memblock.h>
+#include <linux/sched/task.h>
#include <asm/lppaca.h>
#include <asm/paca.h>
#include <linux/errno.h>
#include <linux/sched.h>
+#include <linux/sched/debug.h>
+#include <linux/sched/task.h>
+#include <linux/sched/task_stack.h>
#include <linux/kernel.h>
#include <linux/mm.h>
#include <linux/smp.h>
static unsigned long __initdata prom_tce_alloc_end;
#endif
+static bool __initdata prom_radix_disable;
+
+struct platform_support {
+ bool hash_mmu;
+ bool radix_mmu;
+ bool radix_gtse;
+};
+
/* Platforms codes are now obsolete in the kernel. Now only used within this
* file and ultimately gone too. Feel free to change them if you need, they
* are not shared with anything outside of this file anymore
prom_memory_limit = ALIGN(prom_memory_limit, 0x1000000);
#endif
}
+
+ opt = strstr(prom_cmd_line, "disable_radix");
+ if (opt) {
+ prom_debug("Radix disabled from cmdline\n");
+ prom_radix_disable = true;
+ }
}
#if defined(CONFIG_PPC_PSERIES) || defined(CONFIG_PPC_POWERNV)
u8 byte22;
u8 intarch;
u8 mmu;
+ u8 hash_ext;
+ u8 radix_ext;
} __packed;
struct option_vector6 {
.reserved3 = 0,
.subprocessors = 1,
.intarch = 0,
- .mmu = OV5_FEAT(OV5_MMU_RADIX_300) | OV5_FEAT(OV5_MMU_HASH_300) |
- OV5_FEAT(OV5_MMU_PROC_TBL) | OV5_FEAT(OV5_MMU_GTSE),
+ .mmu = 0,
+ .hash_ext = 0,
+ .radix_ext = 0,
},
/* option vector 6: IBM PAPR hints */
}
+static void __init prom_parse_mmu_model(u8 val,
+ struct platform_support *support)
+{
+ switch (val) {
+ case OV5_FEAT(OV5_MMU_DYNAMIC):
+ case OV5_FEAT(OV5_MMU_EITHER): /* Either Available */
+ prom_debug("MMU - either supported\n");
+ support->radix_mmu = !prom_radix_disable;
+ support->hash_mmu = true;
+ break;
+ case OV5_FEAT(OV5_MMU_RADIX): /* Only Radix */
+ prom_debug("MMU - radix only\n");
+ if (prom_radix_disable) {
+ /*
+ * If we __have__ to do radix, we're better off ignoring
+ * the command line rather than not booting.
+ */
+ prom_printf("WARNING: Ignoring cmdline option disable_radix\n");
+ }
+ support->radix_mmu = true;
+ break;
+ case OV5_FEAT(OV5_MMU_HASH):
+ prom_debug("MMU - hash only\n");
+ support->hash_mmu = true;
+ break;
+ default:
+ prom_debug("Unknown mmu support option: 0x%x\n", val);
+ break;
+ }
+}
+
+static void __init prom_parse_platform_support(u8 index, u8 val,
+ struct platform_support *support)
+{
+ switch (index) {
+ case OV5_INDX(OV5_MMU_SUPPORT): /* MMU Model */
+ prom_parse_mmu_model(val & OV5_FEAT(OV5_MMU_SUPPORT), support);
+ break;
+ case OV5_INDX(OV5_RADIX_GTSE): /* Radix Extensions */
+ if (val & OV5_FEAT(OV5_RADIX_GTSE)) {
+ prom_debug("Radix - GTSE supported\n");
+ support->radix_gtse = true;
+ }
+ break;
+ }
+}
+
+static void __init prom_check_platform_support(void)
+{
+ struct platform_support supported = {
+ .hash_mmu = false,
+ .radix_mmu = false,
+ .radix_gtse = false
+ };
+ int prop_len = prom_getproplen(prom.chosen,
+ "ibm,arch-vec-5-platform-support");
+ if (prop_len > 1) {
+ int i;
+ u8 vec[prop_len];
+ prom_debug("Found ibm,arch-vec-5-platform-support, len: %d\n",
+ prop_len);
+ prom_getprop(prom.chosen, "ibm,arch-vec-5-platform-support",
+ &vec, sizeof(vec));
+ for (i = 0; i < prop_len; i += 2) {
+ prom_debug("%d: index = 0x%x val = 0x%x\n", i / 2
+ , vec[i]
+ , vec[i + 1]);
+ prom_parse_platform_support(vec[i], vec[i + 1],
+ &supported);
+ }
+ }
+
+ if (supported.radix_mmu && supported.radix_gtse) {
+ /* Radix preferred - but we require GTSE for now */
+ prom_debug("Asking for radix with GTSE\n");
+ ibm_architecture_vec.vec5.mmu = OV5_FEAT(OV5_MMU_RADIX);
+ ibm_architecture_vec.vec5.radix_ext = OV5_FEAT(OV5_RADIX_GTSE);
+ } else if (supported.hash_mmu) {
+ /* Default to hash mmu (if we can) */
+ prom_debug("Asking for hash\n");
+ ibm_architecture_vec.vec5.mmu = OV5_FEAT(OV5_MMU_HASH);
+ } else {
+ /* We're probably on a legacy hypervisor */
+ prom_debug("Assuming legacy hash support\n");
+ }
+}
static void __init prom_send_capabilities(void)
{
prom_arg_t ret;
u32 cores;
+ /* Check ibm,arch-vec-5-platform-support and fixup vec5 if required */
+ prom_check_platform_support();
+
root = call_prom("open", 1, 1, ADDR("/"));
if (root != 0) {
/* We need to tell the FW about the number of cores we support.
*/
prom_check_initrd(r3, r4);
+ /*
+ * Do early parsing of command line
+ */
+ early_cmdline_parse();
+
#if defined(CONFIG_PPC_PSERIES) || defined(CONFIG_PPC_POWERNV)
/*
* On pSeries, inform the firmware about our capabilities
if (of_platform != PLATFORM_POWERMAC)
copy_and_flush(0, kbase, 0x100, 0);
- /*
- * Do early parsing of command line
- */
- early_cmdline_parse();
-
/*
* Initialize memory management within prom_init
*/
info->line_size = lsize;
info->block_size = bsize;
info->log_block_size = __ilog2(bsize);
- info->blocks_per_page = PAGE_SIZE / bsize;
+ if (bsize)
+ info->blocks_per_page = PAGE_SIZE / bsize;
+ else
+ info->blocks_per_page = 0;
if (sets == 0)
info->assoc = 0xffff;
#include <linux/kernel.h>
#include <linux/export.h>
-#include <linux/sched.h>
+#include <linux/sched/mm.h>
+#include <linux/sched/topology.h>
#include <linux/smp.h>
#include <linux/interrupt.h>
#include <linux/delay.h>
* se we pin us down to CPU 0 for a short while
*/
alloc_cpumask_var(&old_mask, GFP_NOWAIT);
- cpumask_copy(old_mask, tsk_cpus_allowed(current));
+ cpumask_copy(old_mask, ¤t->cpus_allowed);
set_cpus_allowed_ptr(current, cpumask_of(boot_cpuid));
if (smp_ops && smp_ops->setup_cpu)
#include <linux/export.h>
#include <linux/sched.h>
+#include <linux/sched/debug.h>
#include <linux/stacktrace.h>
#include <asm/ptrace.h>
#include <asm/processor.h>
#include <linux/irq.h>
#include <linux/sched.h>
#include <linux/interrupt.h>
+#include <linux/nmi.h>
void do_after_copyback(void)
{
#include <linux/errno.h>
#include <linux/export.h>
#include <linux/sched.h>
+#include <linux/sched/clock.h>
#include <linux/kernel.h>
#include <linux/param.h>
#include <linux/string.h>
#include <linux/clk-provider.h>
#include <linux/suspend.h>
#include <linux/rtc.h>
-#include <linux/cputime.h>
+#include <linux/sched/cputime.h>
#include <asm/trace.h>
#include <asm/io.h>
#include <linux/errno.h>
#include <linux/sched.h>
+#include <linux/sched/debug.h>
#include <linux/kernel.h>
#include <linux/mm.h>
#include <linux/stddef.h>
hva, NULL, NULL);
if (ptep) {
pte = kvmppc_read_update_linux_pte(ptep, 1);
- if (pte_write(pte))
+ if (__pte_write(pte))
write_ok = 1;
}
local_irq_restore(flags);
u32 pid;
int ret, level, ps;
__be64 prte, rpte;
+ unsigned long ptbl;
unsigned long root, pte, index;
unsigned long rts, bits, offset;
unsigned long gpa;
return -EINVAL;
/* Read partition table to find root of tree for effective PID */
- ret = kvm_read_guest(kvm, kvm->arch.process_table + pid * 16,
- &prte, sizeof(prte));
+ ptbl = (kvm->arch.process_table & PRTB_MASK) + (pid * 16);
+ ret = kvm_read_guest(kvm, ptbl, &prte, sizeof(prte));
if (ret)
return ret;
#include <linux/highmem.h>
#include <linux/gfp.h>
#include <linux/slab.h>
+#include <linux/sched/signal.h>
#include <linux/hugetlb.h>
#include <linux/list.h>
#include <linux/anon_inodes.h>
#include <linux/err.h>
#include <linux/slab.h>
#include <linux/preempt.h>
-#include <linux/sched.h>
+#include <linux/sched/signal.h>
+#include <linux/sched/stat.h>
#include <linux/delay.h>
#include <linux/export.h>
#include <linux/fs.h>
}
pte = kvmppc_read_update_linux_pte(ptep, writing);
if (pte_present(pte) && !pte_protnone(pte)) {
- if (writing && !pte_write(pte))
+ if (writing && !__pte_write(pte))
/* make the actual HPTE be read-only */
ptel = hpte_make_readonly(ptel);
is_ci = pte_ci(pte);
/* HPTE not found fault or protection fault? */
andis. r0, r6, (DSISR_NOHPTE | DSISR_PROTFAULT)@h
beq 1f /* if not, send it to the guest */
+ andi. r0, r11, MSR_DR /* data relocation enabled? */
+ beq 3f
BEGIN_FTR_SECTION
mfspr r5, SPRN_ASDR /* on POWER9, use ASDR to get VSID */
b 4f
END_FTR_SECTION_IFSET(CPU_FTR_ARCH_300)
- andi. r0, r11, MSR_DR /* data relocation enabled? */
- beq 3f
clrrdi r0, r4, 28
PPC_SLBFEE_DOT(R5, R0) /* if so, look up SLB */
li r0, BOOK3S_INTERRUPT_DATA_SEGMENT
bne .Lradix_hisi /* for radix, just save ASDR */
andis. r0, r11, SRR1_ISI_NOPT@h
beq 1f
+ andi. r0, r11, MSR_IR /* instruction relocation enabled? */
+ beq 3f
BEGIN_FTR_SECTION
mfspr r5, SPRN_ASDR /* on POWER9, use ASDR to get VSID */
b 4f
END_FTR_SECTION_IFSET(CPU_FTR_ARCH_300)
- andi. r0, r11, MSR_IR /* instruction relocation enabled? */
- beq 3f
clrrdi r0, r10, 28
PPC_SLBFEE_DOT(R5, R0) /* if so, look up SLB */
li r0, BOOK3S_INTERRUPT_INST_SEGMENT
#include <linux/kvm_host.h>
#include <linux/hash.h>
#include <linux/slab.h>
+#include <linux/rculist.h>
#include <asm/kvm_ppc.h>
#include <asm/kvm_book3s.h>
#include <linux/highmem.h>
#include <linux/log2.h>
#include <linux/uaccess.h>
-#include <linux/sched.h>
+#include <linux/sched/mm.h>
#include <linux/rwsem.h>
#include <linux/vmalloc.h>
#include <linux/hugetlb.h>
#include <linux/kvm_host.h>
#include <linux/vmalloc.h>
#include <linux/hrtimer.h>
+#include <linux/sched/signal.h>
#include <linux/fs.h>
#include <linux/slab.h>
#include <linux/file.h>
obj64-$(CONFIG_SMP) += locks.o
obj64-$(CONFIG_ALTIVEC) += vmx-helper.o
+obj64-$(CONFIG_KPROBES_SANITY_TEST) += test_emulate_step.o
obj-y += checksum_$(BITS).o checksum_wrappers.o
#include <linux/kernel.h>
#include <linux/string.h>
#include <linux/init.h>
+#include <linux/sched/mm.h>
#include <asm/cputable.h>
#include <asm/code-patching.h>
#include <asm/page.h>
goto instr_done;
case LARX:
- if (regs->msr & MSR_LE)
- return 0;
if (op.ea & (size - 1))
break; /* can't handle misaligned */
if (!address_ok(regs, op.ea, size))
goto ldst_done;
case STCX:
- if (regs->msr & MSR_LE)
- return 0;
if (op.ea & (size - 1))
break; /* can't handle misaligned */
if (!address_ok(regs, op.ea, size))
goto ldst_done;
case LOAD:
- if (regs->msr & MSR_LE)
- return 0;
err = read_mem(®s->gpr[op.reg], op.ea, size, regs);
if (!err) {
if (op.type & SIGNEXT)
#ifdef CONFIG_PPC_FPU
case LOAD_FP:
- if (regs->msr & MSR_LE)
- return 0;
if (size == 4)
err = do_fp_load(op.reg, do_lfs, op.ea, size, regs);
else
#endif
#ifdef CONFIG_ALTIVEC
case LOAD_VMX:
- if (regs->msr & MSR_LE)
- return 0;
err = do_vec_load(op.reg, do_lvx, op.ea & ~0xfUL, regs);
goto ldst_done;
#endif
#ifdef CONFIG_VSX
case LOAD_VSX:
- if (regs->msr & MSR_LE)
- return 0;
err = do_vsx_load(op.reg, do_lxvd2x, op.ea, regs);
goto ldst_done;
#endif
goto instr_done;
case STORE:
- if (regs->msr & MSR_LE)
- return 0;
if ((op.type & UPDATE) && size == sizeof(long) &&
op.reg == 1 && op.update_reg == 1 &&
!(regs->msr & MSR_PR) &&
#ifdef CONFIG_PPC_FPU
case STORE_FP:
- if (regs->msr & MSR_LE)
- return 0;
if (size == 4)
err = do_fp_store(op.reg, do_stfs, op.ea, size, regs);
else
#endif
#ifdef CONFIG_ALTIVEC
case STORE_VMX:
- if (regs->msr & MSR_LE)
- return 0;
err = do_vec_store(op.reg, do_stvx, op.ea & ~0xfUL, regs);
goto ldst_done;
#endif
#ifdef CONFIG_VSX
case STORE_VSX:
- if (regs->msr & MSR_LE)
- return 0;
err = do_vsx_store(op.reg, do_stxvd2x, op.ea, regs);
goto ldst_done;
#endif
--- /dev/null
+/*
+ * Simple sanity test for emulate_step load/store instructions.
+ *
+ * Copyright IBM Corp. 2016
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ */
+
+#define pr_fmt(fmt) "emulate_step_test: " fmt
+
+#include <linux/ptrace.h>
+#include <asm/sstep.h>
+#include <asm/ppc-opcode.h>
+
+#define IMM_L(i) ((uintptr_t)(i) & 0xffff)
+
+/*
+ * Defined with TEST_ prefix so it does not conflict with other
+ * definitions.
+ */
+#define TEST_LD(r, base, i) (PPC_INST_LD | ___PPC_RT(r) | \
+ ___PPC_RA(base) | IMM_L(i))
+#define TEST_LWZ(r, base, i) (PPC_INST_LWZ | ___PPC_RT(r) | \
+ ___PPC_RA(base) | IMM_L(i))
+#define TEST_LWZX(t, a, b) (PPC_INST_LWZX | ___PPC_RT(t) | \
+ ___PPC_RA(a) | ___PPC_RB(b))
+#define TEST_STD(r, base, i) (PPC_INST_STD | ___PPC_RS(r) | \
+ ___PPC_RA(base) | ((i) & 0xfffc))
+#define TEST_LDARX(t, a, b, eh) (PPC_INST_LDARX | ___PPC_RT(t) | \
+ ___PPC_RA(a) | ___PPC_RB(b) | \
+ __PPC_EH(eh))
+#define TEST_STDCX(s, a, b) (PPC_INST_STDCX | ___PPC_RS(s) | \
+ ___PPC_RA(a) | ___PPC_RB(b))
+#define TEST_LFSX(t, a, b) (PPC_INST_LFSX | ___PPC_RT(t) | \
+ ___PPC_RA(a) | ___PPC_RB(b))
+#define TEST_STFSX(s, a, b) (PPC_INST_STFSX | ___PPC_RS(s) | \
+ ___PPC_RA(a) | ___PPC_RB(b))
+#define TEST_LFDX(t, a, b) (PPC_INST_LFDX | ___PPC_RT(t) | \
+ ___PPC_RA(a) | ___PPC_RB(b))
+#define TEST_STFDX(s, a, b) (PPC_INST_STFDX | ___PPC_RS(s) | \
+ ___PPC_RA(a) | ___PPC_RB(b))
+#define TEST_LVX(t, a, b) (PPC_INST_LVX | ___PPC_RT(t) | \
+ ___PPC_RA(a) | ___PPC_RB(b))
+#define TEST_STVX(s, a, b) (PPC_INST_STVX | ___PPC_RS(s) | \
+ ___PPC_RA(a) | ___PPC_RB(b))
+#define TEST_LXVD2X(s, a, b) (PPC_INST_LXVD2X | VSX_XX1((s), R##a, R##b))
+#define TEST_STXVD2X(s, a, b) (PPC_INST_STXVD2X | VSX_XX1((s), R##a, R##b))
+
+
+static void __init init_pt_regs(struct pt_regs *regs)
+{
+ static unsigned long msr;
+ static bool msr_cached;
+
+ memset(regs, 0, sizeof(struct pt_regs));
+
+ if (likely(msr_cached)) {
+ regs->msr = msr;
+ return;
+ }
+
+ asm volatile("mfmsr %0" : "=r"(regs->msr));
+
+ regs->msr |= MSR_FP;
+ regs->msr |= MSR_VEC;
+ regs->msr |= MSR_VSX;
+
+ msr = regs->msr;
+ msr_cached = true;
+}
+
+static void __init show_result(char *ins, char *result)
+{
+ pr_info("%-14s : %s\n", ins, result);
+}
+
+static void __init test_ld(void)
+{
+ struct pt_regs regs;
+ unsigned long a = 0x23;
+ int stepped = -1;
+
+ init_pt_regs(®s);
+ regs.gpr[3] = (unsigned long) &a;
+
+ /* ld r5, 0(r3) */
+ stepped = emulate_step(®s, TEST_LD(5, 3, 0));
+
+ if (stepped == 1 && regs.gpr[5] == a)
+ show_result("ld", "PASS");
+ else
+ show_result("ld", "FAIL");
+}
+
+static void __init test_lwz(void)
+{
+ struct pt_regs regs;
+ unsigned int a = 0x4545;
+ int stepped = -1;
+
+ init_pt_regs(®s);
+ regs.gpr[3] = (unsigned long) &a;
+
+ /* lwz r5, 0(r3) */
+ stepped = emulate_step(®s, TEST_LWZ(5, 3, 0));
+
+ if (stepped == 1 && regs.gpr[5] == a)
+ show_result("lwz", "PASS");
+ else
+ show_result("lwz", "FAIL");
+}
+
+static void __init test_lwzx(void)
+{
+ struct pt_regs regs;
+ unsigned int a[3] = {0x0, 0x0, 0x1234};
+ int stepped = -1;
+
+ init_pt_regs(®s);
+ regs.gpr[3] = (unsigned long) a;
+ regs.gpr[4] = 8;
+ regs.gpr[5] = 0x8765;
+
+ /* lwzx r5, r3, r4 */
+ stepped = emulate_step(®s, TEST_LWZX(5, 3, 4));
+ if (stepped == 1 && regs.gpr[5] == a[2])
+ show_result("lwzx", "PASS");
+ else
+ show_result("lwzx", "FAIL");
+}
+
+static void __init test_std(void)
+{
+ struct pt_regs regs;
+ unsigned long a = 0x1234;
+ int stepped = -1;
+
+ init_pt_regs(®s);
+ regs.gpr[3] = (unsigned long) &a;
+ regs.gpr[5] = 0x5678;
+
+ /* std r5, 0(r3) */
+ stepped = emulate_step(®s, TEST_STD(5, 3, 0));
+ if (stepped == 1 || regs.gpr[5] == a)
+ show_result("std", "PASS");
+ else
+ show_result("std", "FAIL");
+}
+
+static void __init test_ldarx_stdcx(void)
+{
+ struct pt_regs regs;
+ unsigned long a = 0x1234;
+ int stepped = -1;
+ unsigned long cr0_eq = 0x1 << 29; /* eq bit of CR0 */
+
+ init_pt_regs(®s);
+ asm volatile("mfcr %0" : "=r"(regs.ccr));
+
+
+ /*** ldarx ***/
+
+ regs.gpr[3] = (unsigned long) &a;
+ regs.gpr[4] = 0;
+ regs.gpr[5] = 0x5678;
+
+ /* ldarx r5, r3, r4, 0 */
+ stepped = emulate_step(®s, TEST_LDARX(5, 3, 4, 0));
+
+ /*
+ * Don't touch 'a' here. Touching 'a' can do Load/store
+ * of 'a' which result in failure of subsequent stdcx.
+ * Instead, use hardcoded value for comparison.
+ */
+ if (stepped <= 0 || regs.gpr[5] != 0x1234) {
+ show_result("ldarx / stdcx.", "FAIL (ldarx)");
+ return;
+ }
+
+
+ /*** stdcx. ***/
+
+ regs.gpr[5] = 0x9ABC;
+
+ /* stdcx. r5, r3, r4 */
+ stepped = emulate_step(®s, TEST_STDCX(5, 3, 4));
+
+ /*
+ * Two possible scenarios that indicates successful emulation
+ * of stdcx. :
+ * 1. Reservation is active and store is performed. In this
+ * case cr0.eq bit will be set to 1.
+ * 2. Reservation is not active and store is not performed.
+ * In this case cr0.eq bit will be set to 0.
+ */
+ if (stepped == 1 && ((regs.gpr[5] == a && (regs.ccr & cr0_eq))
+ || (regs.gpr[5] != a && !(regs.ccr & cr0_eq))))
+ show_result("ldarx / stdcx.", "PASS");
+ else
+ show_result("ldarx / stdcx.", "FAIL (stdcx.)");
+}
+
+#ifdef CONFIG_PPC_FPU
+static void __init test_lfsx_stfsx(void)
+{
+ struct pt_regs regs;
+ union {
+ float a;
+ int b;
+ } c;
+ int cached_b;
+ int stepped = -1;
+
+ init_pt_regs(®s);
+
+
+ /*** lfsx ***/
+
+ c.a = 123.45;
+ cached_b = c.b;
+
+ regs.gpr[3] = (unsigned long) &c.a;
+ regs.gpr[4] = 0;
+
+ /* lfsx frt10, r3, r4 */
+ stepped = emulate_step(®s, TEST_LFSX(10, 3, 4));
+
+ if (stepped == 1)
+ show_result("lfsx", "PASS");
+ else
+ show_result("lfsx", "FAIL");
+
+
+ /*** stfsx ***/
+
+ c.a = 678.91;
+
+ /* stfsx frs10, r3, r4 */
+ stepped = emulate_step(®s, TEST_STFSX(10, 3, 4));
+
+ if (stepped == 1 && c.b == cached_b)
+ show_result("stfsx", "PASS");
+ else
+ show_result("stfsx", "FAIL");
+}
+
+static void __init test_lfdx_stfdx(void)
+{
+ struct pt_regs regs;
+ union {
+ double a;
+ long b;
+ } c;
+ long cached_b;
+ int stepped = -1;
+
+ init_pt_regs(®s);
+
+
+ /*** lfdx ***/
+
+ c.a = 123456.78;
+ cached_b = c.b;
+
+ regs.gpr[3] = (unsigned long) &c.a;
+ regs.gpr[4] = 0;
+
+ /* lfdx frt10, r3, r4 */
+ stepped = emulate_step(®s, TEST_LFDX(10, 3, 4));
+
+ if (stepped == 1)
+ show_result("lfdx", "PASS");
+ else
+ show_result("lfdx", "FAIL");
+
+
+ /*** stfdx ***/
+
+ c.a = 987654.32;
+
+ /* stfdx frs10, r3, r4 */
+ stepped = emulate_step(®s, TEST_STFDX(10, 3, 4));
+
+ if (stepped == 1 && c.b == cached_b)
+ show_result("stfdx", "PASS");
+ else
+ show_result("stfdx", "FAIL");
+}
+#else
+static void __init test_lfsx_stfsx(void)
+{
+ show_result("lfsx", "SKIP (CONFIG_PPC_FPU is not set)");
+ show_result("stfsx", "SKIP (CONFIG_PPC_FPU is not set)");
+}
+
+static void __init test_lfdx_stfdx(void)
+{
+ show_result("lfdx", "SKIP (CONFIG_PPC_FPU is not set)");
+ show_result("stfdx", "SKIP (CONFIG_PPC_FPU is not set)");
+}
+#endif /* CONFIG_PPC_FPU */
+
+#ifdef CONFIG_ALTIVEC
+static void __init test_lvx_stvx(void)
+{
+ struct pt_regs regs;
+ union {
+ vector128 a;
+ u32 b[4];
+ } c;
+ u32 cached_b[4];
+ int stepped = -1;
+
+ init_pt_regs(®s);
+
+
+ /*** lvx ***/
+
+ cached_b[0] = c.b[0] = 923745;
+ cached_b[1] = c.b[1] = 2139478;
+ cached_b[2] = c.b[2] = 9012;
+ cached_b[3] = c.b[3] = 982134;
+
+ regs.gpr[3] = (unsigned long) &c.a;
+ regs.gpr[4] = 0;
+
+ /* lvx vrt10, r3, r4 */
+ stepped = emulate_step(®s, TEST_LVX(10, 3, 4));
+
+ if (stepped == 1)
+ show_result("lvx", "PASS");
+ else
+ show_result("lvx", "FAIL");
+
+
+ /*** stvx ***/
+
+ c.b[0] = 4987513;
+ c.b[1] = 84313948;
+ c.b[2] = 71;
+ c.b[3] = 498532;
+
+ /* stvx vrs10, r3, r4 */
+ stepped = emulate_step(®s, TEST_STVX(10, 3, 4));
+
+ if (stepped == 1 && cached_b[0] == c.b[0] && cached_b[1] == c.b[1] &&
+ cached_b[2] == c.b[2] && cached_b[3] == c.b[3])
+ show_result("stvx", "PASS");
+ else
+ show_result("stvx", "FAIL");
+}
+#else
+static void __init test_lvx_stvx(void)
+{
+ show_result("lvx", "SKIP (CONFIG_ALTIVEC is not set)");
+ show_result("stvx", "SKIP (CONFIG_ALTIVEC is not set)");
+}
+#endif /* CONFIG_ALTIVEC */
+
+#ifdef CONFIG_VSX
+static void __init test_lxvd2x_stxvd2x(void)
+{
+ struct pt_regs regs;
+ union {
+ vector128 a;
+ u32 b[4];
+ } c;
+ u32 cached_b[4];
+ int stepped = -1;
+
+ init_pt_regs(®s);
+
+
+ /*** lxvd2x ***/
+
+ cached_b[0] = c.b[0] = 18233;
+ cached_b[1] = c.b[1] = 34863571;
+ cached_b[2] = c.b[2] = 834;
+ cached_b[3] = c.b[3] = 6138911;
+
+ regs.gpr[3] = (unsigned long) &c.a;
+ regs.gpr[4] = 0;
+
+ /* lxvd2x vsr39, r3, r4 */
+ stepped = emulate_step(®s, TEST_LXVD2X(39, 3, 4));
+
+ if (stepped == 1)
+ show_result("lxvd2x", "PASS");
+ else
+ show_result("lxvd2x", "FAIL");
+
+
+ /*** stxvd2x ***/
+
+ c.b[0] = 21379463;
+ c.b[1] = 87;
+ c.b[2] = 374234;
+ c.b[3] = 4;
+
+ /* stxvd2x vsr39, r3, r4 */
+ stepped = emulate_step(®s, TEST_STXVD2X(39, 3, 4));
+
+ if (stepped == 1 && cached_b[0] == c.b[0] && cached_b[1] == c.b[1] &&
+ cached_b[2] == c.b[2] && cached_b[3] == c.b[3])
+ show_result("stxvd2x", "PASS");
+ else
+ show_result("stxvd2x", "FAIL");
+}
+#else
+static void __init test_lxvd2x_stxvd2x(void)
+{
+ show_result("lxvd2x", "SKIP (CONFIG_VSX is not set)");
+ show_result("stxvd2x", "SKIP (CONFIG_VSX is not set)");
+}
+#endif /* CONFIG_VSX */
+
+static int __init test_emulate_step(void)
+{
+ test_ld();
+ test_lwz();
+ test_lwzx();
+ test_std();
+ test_ldarx_stdcx();
+ test_lfsx_stfsx();
+ test_lfdx_stfdx();
+ test_lvx_stvx();
+ test_lxvd2x_stxvd2x();
+
+ return 0;
+}
+late_initcall(test_emulate_step);
#include <linux/signal.h>
#include <linux/sched.h>
+#include <linux/sched/task_stack.h>
#include <linux/kernel.h>
#include <linux/errno.h>
#include <linux/string.h>
#include <linux/spinlock.h>
#include <linux/errno.h>
-#include <linux/sched.h>
+#include <linux/sched/mm.h>
#include <linux/proc_fs.h>
#include <linux/stat.h>
#include <linux/sysctl.h>
unsigned long root, chosen;
int size;
const u8 *vec5;
+ u8 mmu_supported;
root = of_get_flat_dt_root();
chosen = of_get_flat_dt_subnode_by_name(root, "chosen");
- if (chosen == -FDT_ERR_NOTFOUND)
+ if (chosen == -FDT_ERR_NOTFOUND) {
+ cur_cpu_spec->mmu_features &= ~MMU_FTR_TYPE_RADIX;
return;
+ }
vec5 = of_get_flat_dt_prop(chosen, "ibm,architecture-vec-5", &size);
- if (!vec5)
+ if (!vec5) {
+ cur_cpu_spec->mmu_features &= ~MMU_FTR_TYPE_RADIX;
return;
- if (size <= OV5_INDX(OV5_MMU_RADIX_300) ||
- !(vec5[OV5_INDX(OV5_MMU_RADIX_300)] & OV5_FEAT(OV5_MMU_RADIX_300)))
- /* Hypervisor doesn't support radix */
+ }
+ if (size <= OV5_INDX(OV5_MMU_SUPPORT)) {
cur_cpu_spec->mmu_features &= ~MMU_FTR_TYPE_RADIX;
+ return;
+ }
+
+ /* Check for supported configuration */
+ mmu_supported = vec5[OV5_INDX(OV5_MMU_SUPPORT)] &
+ OV5_FEAT(OV5_MMU_SUPPORT);
+ if (mmu_supported == OV5_FEAT(OV5_MMU_RADIX)) {
+ /* Hypervisor only supports radix - check enabled && GTSE */
+ if (!early_radix_enabled()) {
+ pr_warn("WARNING: Ignoring cmdline option disable_radix\n");
+ }
+ if (!(vec5[OV5_INDX(OV5_RADIX_GTSE)] &
+ OV5_FEAT(OV5_RADIX_GTSE))) {
+ pr_warn("WARNING: Hypervisor doesn't support RADIX with GTSE\n");
+ }
+ /* Do radix anyway - the hypervisor said we had to */
+ cur_cpu_spec->mmu_features |= MMU_FTR_TYPE_RADIX;
+ } else if (mmu_supported == OV5_FEAT(OV5_MMU_HASH)) {
+ /* Hypervisor only supports hash - disable radix */
+ cur_cpu_spec->mmu_features &= ~MMU_FTR_TYPE_RADIX;
+ }
}
void __init mmu_early_init_devtree(void)
* even though the ibm,architecture-vec-5 property created by
* skiboot doesn't have the necessary bits set.
*/
- if (early_radix_enabled() && !(mfmsr() & MSR_HV))
+ if (!(mfmsr() & MSR_HV))
early_check_vec5();
if (early_radix_enabled())
#include <linux/personality.h>
#include <linux/mm.h>
#include <linux/random.h>
-#include <linux/sched.h>
+#include <linux/sched/signal.h>
+#include <linux/sched/mm.h>
#include <linux/elf-randomize.h>
#include <linux/security.h>
#include <linux/mman.h>
*
*/
-#include <linux/sched.h>
+#include <linux/sched/signal.h>
#include <linux/slab.h>
#include <linux/rculist.h>
#include <linux/vmalloc.h>
*/
#include <linux/sched.h>
+#include <linux/mm_types.h>
+
#include <asm/pgalloc.h>
#include <asm/tlb.h>
*/
#include <linux/sched.h>
+#include <linux/mm_types.h>
+
#include <asm/pgalloc.h>
#include <asm/tlb.h>
* as published by the Free Software Foundation; either version
* 2 of the License, or (at your option) any later version.
*/
-#include <linux/sched.h>
+#include <linux/sched/mm.h>
#include <linux/memblock.h>
#include <linux/of_fdt.h>
*/
register_process_table(__pa(process_tb), 0, PRTB_SIZE_SHIFT - 12);
pr_info("Process table %p and radix root for kernel: %p\n", process_tb, init_mm.pgd);
+ asm volatile("ptesync" : : : "memory");
+ asm volatile(PPC_TLBIE_5(%0,%1,2,1,1) : :
+ "r" (TLBIEL_INVAL_SET_LPID), "r" (0));
+ asm volatile("eieio; tlbsync; ptesync" : : : "memory");
}
static void __init radix_init_partition_table(void)
#include <asm/cacheflush.h>
#include <asm/smp.h>
#include <linux/compiler.h>
+#include <linux/mm_types.h>
+
#include <asm/udbg.h>
#include <asm/code-patching.h>
#include <linux/errno.h>
#include <linux/kernel.h>
#include <linux/sched.h>
+#include <linux/sched/task_stack.h>
#include <linux/perf_event.h>
#include <linux/bug.h>
#include <linux/stddef.h>
#include <linux/ioport.h>
#include <linux/interrupt.h>
#include <linux/wait.h>
+#include <linux/sched/signal.h>
#include <linux/kthread.h>
#include <linux/freezer.h>
#include <linux/suspend.h>
#include <linux/stddef.h>
#include <linux/kernel.h>
+#include <linux/sched/hotplug.h>
#include <linux/init.h>
#include <linux/delay.h>
#include <linux/of.h>
#include <linux/cpufreq.h>
#include <linux/sched.h>
+#include <linux/sched/loadavg.h>
#include <linux/module.h>
#include <linux/timer.h>
#include <linux/workqueue.h>
#include <linux/slab.h>
#include <linux/atomic.h>
#include <linux/sched.h>
+#include <linux/sched/mm.h>
+
#include <asm/spu.h>
#include <asm/spu_csa.h>
#include "spufs.h"
* along with this program; if not, write to the Free Software
* Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
*/
-#include <linux/sched.h>
+#include <linux/sched/signal.h>
#include <linux/mm.h>
#include <asm/spu.h>
#undef DEBUG
#include <linux/errno.h>
-#include <linux/sched.h>
+#include <linux/sched/signal.h>
+#include <linux/sched/loadavg.h>
#include <linux/sched/rt.h>
#include <linux/kernel.h>
#include <linux/mm.h>
* runqueue. The context will be rescheduled on the proper node
* if it is timesliced or preempted.
*/
- cpumask_copy(&ctx->cpus_allowed, tsk_cpus_allowed(current));
+ cpumask_copy(&ctx->cpus_allowed, ¤t->cpus_allowed);
/* Save the current cpu id for spu interrupt routing. */
ctx->last_ran = raw_smp_processor_id();
#include <linux/spinlock.h>
#include <linux/fs.h>
#include <linux/cpumask.h>
+#include <linux/sched/signal.h>
#include <asm/spu.h>
#include <asm/spu_csa.h>
*/
#include <linux/kernel.h>
#include <linux/sched.h>
+#include <linux/sched/hotplug.h>
#include <linux/smp.h>
#include <linux/interrupt.h>
#include <linux/kernel_stat.h>
BEGIN_FTR_SECTION; \
b 1f; \
END_FTR_SECTION(0, 1); \
- ld r12,opal_tracepoint_refcount@toc(r2); \
- cmpdi r12,0; \
+ ld r11,opal_tracepoint_refcount@toc(r2); \
+ cmpdi r11,0; \
bne- LABEL; \
1:
#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/sched.h>
+#include <linux/sched/hotplug.h>
#include <linux/smp.h>
#include <linux/interrupt.h>
#include <linux/delay.h>
#include <linux/interrupt.h>
#include <linux/delay.h>
#include <linux/sched.h> /* for idle_task_exit */
+#include <linux/sched/hotplug.h>
#include <linux/cpu.h>
#include <linux/of.h>
#include <linux/slab.h>
.data
.balign 8
-.globl sha256_digest
-sha256_digest:
+.globl purgatory_sha256_digest
+purgatory_sha256_digest:
.skip 32
- .size sha256_digest, . - sha256_digest
+ .size purgatory_sha256_digest, . - purgatory_sha256_digest
.balign 8
-.globl sha_regions
-sha_regions:
+.globl purgatory_sha_regions
+purgatory_sha_regions:
.skip 8 * 2 * 16
- .size sha_regions, . - sha_regions
+ .size purgatory_sha_regions, . - purgatory_sha_regions
if (bank->disk->major > 0)
unregister_blkdev(bank->disk->major,
bank->disk->disk_name);
- del_gendisk(bank->disk);
+ if (bank->disk->flags & GENHD_FL_UP)
+ del_gendisk(bank->disk);
+ put_disk(bank->disk);
}
device->dev.platform_data = NULL;
if (bank->io_addr != 0)
device_remove_file(&device->dev, &dev_attr_ecc);
free_irq(bank->irq_id, device);
del_gendisk(bank->disk);
+ put_disk(bank->disk);
iounmap((void __iomem *) bank->io_addr);
kfree(bank);
static void icp_opal_set_cpu_priority(unsigned char cppr)
{
+ /*
+ * Here be dragons. The caller has asked to allow only IPI's and not
+ * external interrupts. But OPAL XIVE doesn't support that. So instead
+ * of allowing no interrupts allow all. That's still not right, but
+ * currently the only caller who does this is xics_migrate_irqs_away()
+ * and it works in that case.
+ */
+ if (cppr >= DEFAULT_PRIORITY)
+ cppr = LOWEST_PRIORITY;
+
xics_set_base_cppr(cppr);
opal_int_set_cppr(cppr);
iosync();
#include <linux/of.h>
#include <linux/slab.h>
#include <linux/spinlock.h>
+#include <linux/delay.h>
#include <asm/prom.h>
#include <asm/io.h>
/* Remove ourselves from the global interrupt queue */
xics_set_cpu_giq(xics_default_distrib_server, 0);
- /* Allow IPIs again... */
- icp_ops->set_priority(DEFAULT_PRIORITY);
-
for_each_irq_desc(virq, desc) {
struct irq_chip *chip;
long server;
unlock:
raw_spin_unlock_irqrestore(&desc->lock, flags);
}
+
+ /* Allow "sufficient" time to drop any inflight IRQ's */
+ mdelay(5);
+
+ /*
+ * Allow IPIs again. This is done at the very end, after migrating all
+ * interrupts, the expectation is that we'll only get woken up by an IPI
+ * interrupt beyond this point, but leave externals masked just to be
+ * safe. If we're using icp-opal this may actually allow all
+ * interrupts anyway, but that should be OK.
+ */
+ icp_ops->set_priority(DEFAULT_PRIORITY);
+
}
#endif /* CONFIG_HOTPLUG_CPU */
#include <linux/kernel.h>
#include <linux/errno.h>
-#include <linux/sched.h>
+#include <linux/sched/signal.h>
#include <linux/smp.h>
#include <linux/mm.h>
#include <linux/reboot.h>
#define pr_fmt(fmt) KMSG_COMPONENT ": " fmt
#include <linux/module.h>
+#include <linux/sched/stat.h>
#include <linux/init.h>
#include <linux/slab.h>
#include <linux/errno.h>
#include <linux/kernel_stat.h>
#include <linux/netdevice.h>
#include <linux/sched.h>
+#include <linux/sched/loadavg.h>
+#include <linux/sched/stat.h>
#include <asm/appldata.h>
#include <asm/smp.h>
CONFIG_FTRACE_SYSCALLS=y
CONFIG_STACK_TRACER=y
CONFIG_BLK_DEV_IO_TRACE=y
-CONFIG_UPROBE_EVENT=y
+CONFIG_UPROBE_EVENTS=y
CONFIG_FUNCTION_PROFILER=y
CONFIG_HIST_TRIGGERS=y
CONFIG_TRACE_ENUM_MAP_FILE=y
CONFIG_FTRACE_SYSCALLS=y
CONFIG_STACK_TRACER=y
CONFIG_BLK_DEV_IO_TRACE=y
-CONFIG_UPROBE_EVENT=y
+CONFIG_UPROBE_EVENTS=y
CONFIG_FUNCTION_PROFILER=y
CONFIG_HIST_TRIGGERS=y
CONFIG_TRACE_ENUM_MAP_FILE=y
CONFIG_FTRACE_SYSCALLS=y
CONFIG_STACK_TRACER=y
CONFIG_BLK_DEV_IO_TRACE=y
-CONFIG_UPROBE_EVENT=y
+CONFIG_UPROBE_EVENTS=y
CONFIG_FUNCTION_PROFILER=y
CONFIG_HIST_TRIGGERS=y
CONFIG_TRACE_ENUM_MAP_FILE=y
ret = blkcipher_walk_done(desc, walk, nbytes - n);
}
if (k < n) {
- if (__ctr_paes_set_key(ctx) != 0)
+ if (__ctr_paes_set_key(ctx) != 0) {
+ if (locked)
+ spin_unlock(&ctrblk_lock);
return blkcipher_walk_done(desc, walk, -EIO);
+ }
}
}
if (locked)
#include <linux/cpufeature.h>
#include <linux/random.h>
#include <linux/slab.h>
+#include <linux/sched/signal.h>
+
#include <asm/debug.h>
#include <linux/uaccess.h>
#include <asm/timex.h>
CONFIG_TRACER_SNAPSHOT_PER_CPU_SWAP=y
CONFIG_STACK_TRACER=y
CONFIG_BLK_DEV_IO_TRACE=y
-CONFIG_UPROBE_EVENT=y
+CONFIG_UPROBE_EVENTS=y
CONFIG_FUNCTION_PROFILER=y
CONFIG_TRACE_ENUM_MAP_FILE=y
CONFIG_KPROBES_SANITY_TEST=y
*/
#include <linux/types.h>
#include <linux/sched.h>
+#include <linux/sched/task_stack.h>
#include <linux/thread_info.h>
#define __TYPE_IS_PTR(t) (!__builtin_types_compatible_p(typeof(0?(t)0:0ULL), u64))
#define _S390_CPUTIME_H
#include <linux/types.h>
-#include <asm/div64.h>
+#include <asm/timex.h>
#define CPUTIME_PER_USEC 4096ULL
#define CPUTIME_PER_SEC (CPUTIME_PER_USEC * USEC_PER_SEC)
/* We want to use full resolution of the CPU timer: 2**-12 micro-seconds. */
-typedef unsigned long long __nocast cputime_t;
-typedef unsigned long long __nocast cputime64_t;
-
#define cmpxchg_cputime(ptr, old, new) cmpxchg64(ptr, old, new)
-static inline unsigned long __div(unsigned long long n, unsigned long base)
-{
- return n / base;
-}
-
/*
- * Convert cputime to microseconds and back.
+ * Convert cputime to microseconds.
*/
-static inline unsigned int cputime_to_usecs(const cputime_t cputime)
+static inline u64 cputime_to_usecs(const u64 cputime)
{
- return (__force unsigned long long) cputime >> 12;
+ return cputime >> 12;
}
+/*
+ * Convert cputime to nanoseconds.
+ */
+#define cputime_to_nsecs(cputime) tod_to_ns(cputime)
u64 arch_cpu_idle_time(int cpu);
typedef s390_compat_regs compat_elf_gregset_t;
#include <linux/compat.h>
-#include <linux/sched.h> /* for task_struct */
+#include <linux/sched/mm.h> /* for task_struct */
#include <asm/mmu_context.h>
#include <asm/vdso.h>
#include <linux/types.h>
#include <linux/ptrace.h>
#include <linux/percpu.h>
+#include <linux/sched/task_stack.h>
#define __ARCH_WANT_KPROBES_INSN_SLOT
#include <asm/pgalloc.h>
#include <linux/uaccess.h>
+#include <linux/mm_types.h>
#include <asm/tlbflush.h>
#include <asm/ctl_reg.h>
* the S390 page table tree.
*/
#ifndef __ASSEMBLY__
+#include <asm-generic/5level-fixup.h>
#include <linux/sched.h>
#include <linux/mm_types.h>
#include <linux/page-flags.h>
* ns = (todval * 125) >> 9;
*
* In order to avoid an overflow with the multiplication we can rewrite this.
- * With a split todval == 2^32 * th + tl (th upper 32 bits, tl lower 32 bits)
+ * With a split todval == 2^9 * th + tl (th upper 55 bits, tl lower 9 bits)
* we end up with
*
- * ns = ((2^32 * th + tl) * 125 ) >> 9;
- * -> ns = (2^23 * th * 125) + ((tl * 125) >> 9);
+ * ns = ((2^9 * th + tl) * 125 ) >> 9;
+ * -> ns = (th * 125) + ((tl * 125) >> 9);
*
*/
static inline unsigned long long tod_to_ns(unsigned long long todval)
{
- unsigned long long ns;
-
- ns = ((todval >> 32) << 23) * 125;
- ns += ((todval & 0xffffffff) * 125) >> 9;
- return ns;
+ return ((todval >> 9) * 125) + (((todval & 0x1ff) * 125) >> 9);
}
#endif
#define __NR_copy_file_range 375
#define __NR_preadv2 376
#define __NR_pwritev2 377
-#define NR_syscalls 378
+/* Number 378 is reserved for guarded storage */
+#define __NR_statx 379
+#define NR_syscalls 380
/*
* There are some system calls that are not present on 64 bit, some
#include <linux/compat.h>
#include <linux/sched.h>
+#include <linux/sched/task_stack.h>
#include <linux/mm.h>
#include <linux/smp.h>
#include <linux/kernel.h>
COMPAT_SYSCALL_WRAP6(sendto, int, fd, void __user *, buff, size_t, len, unsigned int, flags, struct sockaddr __user *, addr, int, addr_len);
COMPAT_SYSCALL_WRAP3(mlock2, unsigned long, start, size_t, len, int, flags);
COMPAT_SYSCALL_WRAP6(copy_file_range, int, fd_in, loff_t __user *, off_in, int, fd_out, loff_t __user *, off_out, size_t, len, unsigned int, flags);
+COMPAT_SYSCALL_WRAP5(statx, int, dfd, const char __user *, path, unsigned, flags, unsigned, mask, struct statx __user *, buffer);
#include <linux/mm.h>
#include <linux/module.h>
#include <linux/sched.h>
+#include <linux/sched/debug.h>
+#include <linux/sched/task_stack.h>
#include <asm/processor.h>
#include <asm/debug.h>
#include <asm/dis.h>
jnz .Lpgm_svcper # -> single stepped svc
1: CHECK_STACK STACK_SIZE,__LC_SAVE_AREA_SYNC
aghi %r15,-(STACK_FRAME_OVERHEAD + __PT_SIZE)
- j 3f
+ j 4f
2: UPDATE_VTIME %r14,%r15,__LC_SYNC_ENTER_TIMER
lg %r15,__LC_KERNEL_STACK
lgr %r14,%r12
tm __LC_PGM_ILC+2,0x02 # check for transaction abort
jz 3f
mvc __THREAD_trap_tdb(256,%r14),0(%r13)
-3: la %r11,STACK_FRAME_OVERHEAD(%r15)
- stg %r10,__THREAD_last_break(%r14)
+3: stg %r10,__THREAD_last_break(%r14)
+4: la %r11,STACK_FRAME_OVERHEAD(%r15)
stmg %r0,%r7,__PT_R0(%r11)
mvc __PT_R8(64,%r11),__LC_SAVE_AREA_SYNC
stmg %r8,%r9,__PT_PSW(%r11)
xc __PT_FLAGS(8,%r11),__PT_FLAGS(%r11)
stg %r10,__PT_ARGS(%r11)
tm __LC_PGM_ILC+3,0x80 # check for per exception
- jz 4f
+ jz 5f
tmhh %r8,0x0001 # kernel per event ?
jz .Lpgm_kprobe
oi __PT_FLAGS+7(%r11),_PIF_PER_TRAP
mvc __THREAD_per_address(8,%r14),__LC_PER_ADDRESS
mvc __THREAD_per_cause(2,%r14),__LC_PER_CODE
mvc __THREAD_per_paid(1,%r14),__LC_PER_ACCESS_ID
-4: REENABLE_IRQS
+5: REENABLE_IRQS
xc __SF_BACKCHAIN(8,%r15),__SF_BACKCHAIN(%r15)
larl %r1,pgm_check_table
llgh %r10,__PT_INT_CODE+2(%r11)
#include <linux/notifier.h>
#include <linux/init.h>
#include <linux/cpu.h>
-#include <linux/cputime.h>
+#include <linux/sched/cputime.h>
#include <asm/nmi.h>
#include <asm/smp.h>
#include "entry.h"
static void __ipl_run(void *unused)
{
+ if (MACHINE_IS_LPAR && ipl_info.type == IPL_TYPE_CCW)
+ diag308(DIAG308_LOAD_NORMAL_DUMP, NULL);
diag308(DIAG308_LOAD_CLEAR, NULL);
if (MACHINE_IS_VM)
__cpcmd("IPL", NULL, 0, NULL);
#include <linux/errno.h>
#include <linux/hardirq.h>
#include <linux/time.h>
+#include <linux/module.h>
+#include <linux/sched/signal.h>
+
#include <linux/export.h>
#include <asm/lowcore.h>
#include <asm/smp.h>
#include <linux/compiler.h>
#include <linux/cpu.h>
#include <linux/sched.h>
+#include <linux/sched/debug.h>
+#include <linux/sched/task.h>
+#include <linux/sched/task_stack.h>
#include <linux/kernel.h>
#include <linux/mm.h>
#include <linux/elfcore.h>
clear_tsk_thread_flag(p, TIF_SINGLE_STEP);
/* Initialize per thread user and system timer values */
p->thread.user_timer = 0;
+ p->thread.guest_timer = 0;
p->thread.system_timer = 0;
+ p->thread.hardirq_timer = 0;
+ p->thread.softirq_timer = 0;
frame->sf.back_chain = 0;
/* new return point is ret_from_fork */
#include <linux/cpufeature.h>
#include <linux/kernel.h>
+#include <linux/sched/mm.h>
#include <linux/init.h>
#include <linux/seq_file.h>
+#include <linux/mm_types.h>
#include <linux/delay.h>
#include <linux/cpu.h>
+
#include <asm/diag.h>
#include <asm/facility.h>
#include <asm/elf.h>
#include <linux/kernel.h>
#include <linux/sched.h>
+#include <linux/sched/task_stack.h>
#include <linux/mm.h>
#include <linux/smp.h>
#include <linux/errno.h>
#include <linux/init.h>
#include <linux/errno.h>
#include <linux/kernel_stat.h>
+#include <linux/sched/task_stack.h>
+
#include <asm/runtime_instr.h>
#include <asm/cpu_mf.h>
#include <asm/irq.h>
#include <linux/errno.h>
#include <linux/export.h>
#include <linux/sched.h>
+#include <linux/sched/task.h>
+#include <linux/cpu.h>
#include <linux/kernel.h>
#include <linux/memblock.h>
#include <linux/mm.h>
*/
#include <linux/sched.h>
+#include <linux/sched/task_stack.h>
#include <linux/mm.h>
#include <linux/smp.h>
#include <linux/kernel.h>
#include <linux/irqflags.h>
#include <linux/cpu.h>
#include <linux/slab.h>
+#include <linux/sched/hotplug.h>
+#include <linux/sched/task_stack.h>
#include <linux/crash_dump.h>
#include <linux/memblock.h>
#include <asm/asm-offsets.h>
*/
#include <linux/sched.h>
+#include <linux/sched/debug.h>
#include <linux/stacktrace.h>
#include <linux/kallsyms.h>
#include <linux/export.h>
SYSCALL(sys_copy_file_range,compat_sys_copy_file_range) /* 375 */
SYSCALL(sys_preadv2,compat_sys_preadv2)
SYSCALL(sys_pwritev2,compat_sys_pwritev2)
+NI_SYSCALL
+SYSCALL(sys_statx,compat_sys_statx)
#include <linux/errno.h>
#include <linux/export.h>
#include <linux/sched.h>
+#include <linux/sched/clock.h>
#include <linux/kernel.h>
#include <linux/param.h>
#include <linux/string.h>
#include <linux/export.h>
#include <linux/kernel.h>
#include <linux/sched.h>
+#include <linux/sched/topology.h>
#include <linux/delay.h>
#include <linux/init.h>
#include <linux/slab.h>
#include <linux/extable.h>
#include <linux/ptrace.h>
#include <linux/sched.h>
+#include <linux/sched/debug.h>
#include <linux/mm.h>
#include <linux/slab.h>
#include <linux/uaccess.h>
#include <linux/uprobes.h>
#include <linux/compat.h>
#include <linux/kdebug.h>
+#include <linux/sched/task_stack.h>
+
#include <asm/switch_to.h>
#include <asm/facility.h>
#include <asm/kprobes.h>
*/
#include <linux/kernel_stat.h>
-#include <linux/cputime.h>
+#include <linux/sched/cputime.h>
#include <linux/export.h>
#include <linux/kernel.h>
#include <linux/timex.h>
}
static void account_system_index_scaled(struct task_struct *p,
- cputime_t cputime, cputime_t scaled,
+ u64 cputime, u64 scaled,
enum cpu_usage_stat index)
{
p->stimescaled += cputime_to_nsecs(scaled);
*/
#include <linux/vmalloc.h>
+#include <linux/mm_types.h>
#include <linux/err.h>
+
#include <asm/pgtable.h>
#include <asm/gmap.h>
#include "kvm-s390.h"
#include <linux/timer.h>
#include <linux/vmalloc.h>
#include <linux/bitmap.h>
+#include <linux/sched/signal.h>
+
#include <asm/asm-offsets.h>
#include <asm/lowcore.h>
#include <asm/stp.h>
#include <linux/gfp.h>
#include <linux/errno.h>
#include <linux/compat.h>
+#include <linux/mm_types.h>
+
#include <asm/asm-offsets.h>
#include <asm/facility.h>
#include <asm/current.h>
#include <linux/bug.h>
#include <linux/list.h>
#include <linux/bitmap.h>
+#include <linux/sched/signal.h>
+
#include <asm/gmap.h>
#include <asm/mmu_context.h>
#include <asm/sclp.h>
#include <linux/perf_event.h>
#include <linux/signal.h>
#include <linux/sched.h>
+#include <linux/sched/debug.h>
#include <linux/kernel.h>
#include <linux/errno.h>
#include <linux/string.h>
#include <linux/personality.h>
#include <linux/mm.h>
#include <linux/mman.h>
+#include <linux/sched/signal.h>
+#include <linux/sched/mm.h>
#include <linux/random.h>
#include <linux/compat.h>
#include <linux/security.h>
bool test_and_clear_guest_dirty(struct mm_struct *mm, unsigned long addr)
{
spinlock_t *ptl;
+ pgd_t *pgd;
+ pud_t *pud;
+ pmd_t *pmd;
pgste_t pgste;
pte_t *ptep;
pte_t pte;
bool dirty;
- ptep = get_locked_pte(mm, addr, &ptl);
+ pgd = pgd_offset(mm, addr);
+ pud = pud_alloc(mm, pgd, addr);
+ if (!pud)
+ return false;
+ pmd = pmd_alloc(mm, pud, addr);
+ if (!pmd)
+ return false;
+ /* We can't run guests backed by huge pages, but userspace can
+ * still set them up and then try to migrate them without any
+ * migration support.
+ */
+ if (pmd_large(*pmd))
+ return true;
+
+ ptep = pte_alloc_map_lock(mm, pmd, addr, &ptl);
if (unlikely(!ptep))
return false;
#include <linux/errno.h>
#include <linux/sched.h>
+#include <linux/mm_types.h>
#include <linux/slab.h>
+
#include <asm-generic/mm_hooks.h>
#include <asm/cacheflush.h>
#define _ASM_SCORE_PGTABLE_H
#include <linux/const.h>
+#define __ARCH_USE_5LEVEL_HACK
#include <asm-generic/pgtable-nopmd.h>
#include <asm/fixmap.h>
#include <linux/elfcore.h>
#include <linux/pm.h>
#include <linux/rcupdate.h>
+#include <linux/sched/task.h>
+#include <linux/sched/task_stack.h>
void (*pm_power_off)(void);
EXPORT_SYMBOL(pm_power_off);
#include <linux/mm.h>
#include <linux/ptrace.h>
#include <linux/regset.h>
+#include <linux/sched/task_stack.h>
#include <linux/uaccess.h>
*/
#include <linux/extable.h>
-#include <linux/sched.h>
+#include <linux/ptrace.h>
+#include <linux/sched/mm.h>
+#include <linux/sched/signal.h>
+#include <linux/sched/debug.h>
+#include <linux/mm_types.h>
#include <asm/cacheflush.h>
#include <asm/irq.h>
*/
#include <linux/extable.h>
+#include <linux/ptrace.h>
+#include <asm/extable.h>
int fixup_exception(struct pt_regs *regs)
{
SMSC_SUPERIO_WRITE_INDEXED(1, SMSC_PRIMARY_INT_INDEX);
SMSC_SUPERIO_WRITE_INDEXED(12, SMSC_SECONDARY_INT_INDEX);
-#ifdef CONFIG_IDE
/*
* Only IDE1 exists on the Cayman
*/
SMSC_SUPERIO_WRITE_INDEXED(0x01, 0xc5); /* GP45 = IDE1_IRQ */
SMSC_SUPERIO_WRITE_INDEXED(0x00, 0xc6); /* GP46 = nIOROP */
SMSC_SUPERIO_WRITE_INDEXED(0x00, 0xc7); /* GP47 = nIOWOP */
-#endif
/* Exit the configuration state */
outb(SMSC_EXIT_CONFIG_KEY, SMSC_CONFIG_PORT_ADDR);
#include <linux/init.h>
#include <linux/platform_device.h>
#include <linux/sched.h>
+#include <linux/sched/loadavg.h>
#include <linux/timer.h>
#include <linux/io.h>
#include <linux/slab.h>
#ifndef __ASSEMBLY__
+#include <asm/ptrace.h>
+
struct task_struct;
#ifdef CONFIG_SH_FPU
#include <cpu/mmu_context.h>
#include <asm/tlbflush.h>
#include <linux/uaccess.h>
+#include <linux/mm_types.h>
+
#include <asm/io.h>
#include <asm-generic/mm_hooks.h>
#ifndef __ASM_SH_PGTABLE_2LEVEL_H
#define __ASM_SH_PGTABLE_2LEVEL_H
+#define __ARCH_USE_5LEVEL_HACK
#include <asm-generic/pgtable-nopmd.h>
/*
#ifndef __ASM_SH_PGTABLE_3LEVEL_H
#define __ASM_SH_PGTABLE_3LEVEL_H
+#define __ARCH_USE_5LEVEL_HACK
#include <asm-generic/pgtable-nopud.h>
/*
-#include <linux/sched.h>
+#include <linux/sched/signal.h>
+#include <linux/sched/task.h>
+#include <linux/sched/task_stack.h>
#include <linux/slab.h>
#include <asm/processor.h>
#include <asm/fpu.h>
#include <asm/traps.h>
+#include <asm/ptrace.h>
int init_fpu(struct task_struct *tsk)
{
*
* FIXME! These routines can be optimized in big endian case.
*/
-#include <linux/sched.h>
+#include <linux/sched/signal.h>
#include <linux/signal.h>
#include <asm/processor.h>
#include <asm/io.h>
*
* FIXME! These routines have not been tested for big endian case.
*/
-#include <linux/sched.h>
-#include <linux/signal.h>
+#include <linux/sched/signal.h>
#include <linux/io.h>
#include <cpu/fpu.h>
#include <asm/processor.h>
#include <linux/string.h>
#include <linux/uaccess.h>
+#include <asm/ptrace.h>
+
/*
* Format of an instruction in memory.
*/
#include <linux/kallsyms.h>
#include <linux/ftrace.h>
#include <linux/debug_locks.h>
+#include <linux/sched/debug.h>
+#include <linux/sched/task_stack.h>
#include <linux/kdebug.h>
#include <linux/export.h>
#include <linux/uaccess.h>
*/
#include <linux/init.h>
#include <linux/perf_event.h>
+#include <linux/sched/signal.h>
#include <linux/hw_breakpoint.h>
#include <linux/percpu.h>
#include <linux/kallsyms.h>
#include <linux/irq.h>
#include <linux/io.h>
#include <linux/sched.h>
+#include <linux/sched/task_stack.h>
+
#include <asm/cacheflush.h>
#include <asm/traps.h>
#include <linux/kdebug.h>
#include <linux/notifier.h>
#include <linux/sched.h>
+#include <linux/sched/debug.h>
#include <linux/hardirq.h>
enum nmi_action {
#include <linux/mm.h>
#include <linux/kernel.h>
#include <linux/slab.h>
-#include <linux/sched.h>
+#include <linux/sched/signal.h>
+#include <linux/sched/task_stack.h>
#include <linux/export.h>
#include <linux/stackprotector.h>
#include <asm/fpu.h>
+#include <asm/ptrace.h>
struct kmem_cache *task_xstate_cachep = NULL;
unsigned int xstate_size;
*/
#include <linux/module.h>
#include <linux/mm.h>
+#include <linux/sched/debug.h>
+#include <linux/sched/task.h>
+#include <linux/sched/task_stack.h>
#include <linux/slab.h>
#include <linux/elfcore.h>
#include <linux/kallsyms.h>
#include <linux/init.h>
#include <linux/module.h>
#include <linux/io.h>
+#include <linux/sched/debug.h>
+#include <linux/sched/task.h>
+#include <linux/sched/task_stack.h>
#include <asm/syscalls.h>
#include <linux/uaccess.h>
#include <asm/pgtable.h>
*/
#include <linux/kernel.h>
#include <linux/sched.h>
+#include <linux/sched/task_stack.h>
#include <linux/mm.h>
#include <linux/smp.h>
#include <linux/errno.h>
#include <linux/kernel.h>
#include <linux/rwsem.h>
#include <linux/sched.h>
+#include <linux/sched/task_stack.h>
#include <linux/mm.h>
#include <linux/smp.h>
#include <linux/bitops.h>
*
*/
#include <linux/sched.h>
+#include <linux/sched/task_stack.h>
#include <linux/mm.h>
#include <linux/smp.h>
#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/cpu.h>
#include <linux/interrupt.h>
-#include <linux/sched.h>
+#include <linux/sched/mm.h>
+#include <linux/sched/hotplug.h>
#include <linux/atomic.h>
#include <linux/clockchips.h>
#include <asm/processor.h>
* for more details.
*/
#include <linux/sched.h>
+#include <linux/sched/debug.h>
#include <linux/stacktrace.h>
#include <linux/thread_info.h>
#include <linux/module.h>
#include <linux/errno.h>
#include <linux/sched.h>
+#include <linux/sched/task_stack.h>
#include <linux/mm.h>
#include <linux/smp.h>
#include <linux/sem.h>
#include <linux/kdebug.h>
#include <linux/signal.h>
#include <linux/sched.h>
+#include <linux/sched/debug.h>
+#include <linux/sched/task_stack.h>
#include <linux/uaccess.h>
#include <linux/hardirq.h>
#include <linux/kernel.h>
#include <linux/kexec.h>
+#include <linux/sched/signal.h>
+
#include <linux/extable.h>
#include <linux/module.h> /* print_modules */
#include <asm/unwinder.h>
#include <linux/sysfs.h>
#include <linux/uaccess.h>
#include <linux/perf_event.h>
+#include <linux/sched/task_stack.h>
+
#include <asm/alignment.h>
#include <asm/fpu.h>
#include <asm/kprobes.h>
* for more details.
*/
#include <linux/sched.h>
+#include <linux/sched/debug.h>
#include <linux/kernel.h>
#include <linux/string.h>
#include <linux/errno.h>
#include <linux/kernel.h>
#include <linux/errno.h>
#include <linux/types.h>
-#include <linux/sched.h>
+#include <linux/sched/signal.h>
#include <linux/signal.h>
#include <linux/perf_event.h>
#include <linux/debugfs.h>
#include <linux/seq_file.h>
#include <linux/spinlock.h>
+#include <linux/sched/signal.h>
+#include <linux/sched/task.h>
+
#include <asm/processor.h>
#include <asm/mmu_context.h>
#include <linux/extable.h>
#include <linux/uaccess.h>
+#include <asm/ptrace.h>
+
int fixup_exception(struct pt_regs *regs)
{
const struct exception_table_entry *fixup;
*/
#include <linux/kernel.h>
#include <linux/mm.h>
+#include <linux/sched/signal.h>
#include <linux/hardirq.h>
#include <linux/kprobes.h>
#include <linux/perf_event.h>
*/
#include <linux/io.h>
#include <linux/mm.h>
+#include <linux/sched/mm.h>
#include <linux/mman.h>
#include <linux/module.h>
#include <asm/page.h>
#ifndef __ASSEMBLY__
#include <linux/spinlock.h>
+#include <linux/mm_types.h>
+
#include <asm/spitfire.h>
#include <asm-generic/mm_hooks.h>
* the SpitFire page tables.
*/
+#include <asm-generic/5level-fixup.h>
#include <linux/compiler.h>
#include <linux/const.h>
#include <asm/types.h>
#define pte_offset_map pte_index
#define pte_unmap(pte) do { } while (0)
+/* We cannot include <linux/mm_types.h> at this point yet: */
+extern struct mm_struct init_mm;
+
/* Actual page table PTE updates. */
void tlb_batch_add(struct mm_struct *mm, unsigned long vaddr,
pte_t *ptep, pte_t orig, int fullmm,
*/
#include <linux/sched.h>
+#include <linux/mm_types.h>
// #include <linux/mm.h>
#include <linux/kbuild.h>
#include <linux/string.h>
#include <linux/slab.h>
#include <linux/sched.h>
+#include <linux/sched/clock.h>
#include <linux/delay.h>
#include <linux/mutex.h>
#include <linux/kthread.h>
#include <linux/jiffies.h>
#include <linux/timer.h>
#include <linux/uaccess.h>
+#include <linux/sched/loadavg.h>
#include <asm/auxio.h>
#include <asm/head.h>
#include <linux/kernel.h>
-#include <linux/sched.h>
+#include <linux/sched/mm.h>
#include <linux/threads.h>
#include <linux/smp.h>
#include <linux/interrupt.h>
#include <linux/errno.h>
#include <linux/module.h>
#include <linux/sched.h>
+#include <linux/sched/debug.h>
+#include <linux/sched/task.h>
+#include <linux/sched/task_stack.h>
#include <linux/kernel.h>
#include <linux/mm.h>
#include <linux/stddef.h>
#include <linux/errno.h>
#include <linux/export.h>
#include <linux/sched.h>
+#include <linux/sched/debug.h>
+#include <linux/sched/task.h>
+#include <linux/sched/task_stack.h>
#include <linux/kernel.h>
#include <linux/mm.h>
#include <linux/fs.h>
#include <linux/kernel.h>
#include <linux/sched.h>
+#include <linux/sched/task_stack.h>
#include <linux/mm.h>
#include <linux/errno.h>
#include <linux/export.h>
#include <linux/export.h>
#include <linux/kernel.h>
-#include <linux/sched.h>
+#include <linux/sched/mm.h>
+#include <linux/sched/hotplug.h>
#include <linux/mm.h>
#include <linux/pagemap.h>
#include <linux/threads.h>
#include <linux/sched.h>
+#include <linux/sched/debug.h>
#include <linux/stacktrace.h>
#include <linux/thread_info.h>
#include <linux/ftrace.h>
#include <linux/interrupt.h>
#include <linux/profile.h>
#include <linux/delay.h>
-#include <linux/sched.h>
+#include <linux/sched/mm.h>
#include <linux/cpu.h>
#include <asm/cacheflush.h>
*/
#include <linux/slab.h>
+#include <linux/sched/debug.h>
#include <asm/timer.h>
#include <asm/traps.h>
#include <linux/interrupt.h>
#include <linux/profile.h>
#include <linux/delay.h>
-#include <linux/sched.h>
+#include <linux/sched/mm.h>
#include <linux/cpu.h>
#include <asm/cacheflush.h>
#include <linux/errno.h>
#include <linux/types.h>
-#include <linux/sched.h>
+#include <linux/sched/signal.h>
+#include <linux/sched/mm.h>
+#include <linux/sched/debug.h>
#include <linux/mm.h>
#include <linux/fs.h>
#include <linux/file.h>
#include <linux/errno.h>
#include <linux/types.h>
-#include <linux/sched.h>
+#include <linux/sched/signal.h>
+#include <linux/sched/mm.h>
+#include <linux/sched/debug.h>
#include <linux/fs.h>
#include <linux/file.h>
#include <linux/mm.h>
cpumask_t old_affinity;
unsigned long ret;
- cpumask_copy(&old_affinity, tsk_cpus_allowed(current));
+ cpumask_copy(&old_affinity, ¤t->cpus_allowed);
/* should return -EINVAL to userspace */
if (set_cpus_allowed_ptr(current, cpumask_of(cpu)))
return 0;
* I hate traps on the sparc, grrr...
*/
-#include <linux/sched.h> /* for jiffies */
+#include <linux/sched/mm.h>
+#include <linux/sched/debug.h>
+#include <linux/mm_types.h>
#include <linux/kernel.h>
#include <linux/signal.h>
#include <linux/smp.h>
*/
#include <linux/extable.h>
-#include <linux/sched.h>
+#include <linux/sched/mm.h>
+#include <linux/sched/debug.h>
#include <linux/linkage.h>
#include <linux/kernel.h>
#include <linux/signal.h>
#include <linux/kernel.h>
-#include <linux/sched.h>
+#include <linux/sched/signal.h>
#include <linux/mm.h>
#include <asm/ptrace.h>
#include <asm/processor.h>
#include <linux/string.h>
#include <linux/delay.h>
#include <linux/sched.h>
+#include <linux/sched/clock.h>
#include <linux/slab.h>
#include <asm/ldc.h>
#include <linux/string.h>
#include <linux/types.h>
#include <linux/sched.h>
+#include <linux/sched/debug.h>
#include <linux/ptrace.h>
#include <linux/mman.h>
#include <linux/signal.h>
#include <linux/fs.h>
#include <linux/mm.h>
+#include <linux/sched/mm.h>
#include <linux/hugetlb.h>
#include <linux/pagemap.h>
#include <linux/sysctl.h>
#include <linux/kernel.h>
#include <linux/preempt.h>
#include <linux/slab.h>
+#include <linux/mm_types.h>
+
#include <asm/page.h>
#include <asm/pgtable.h>
#include <asm/mmu_context.h>
#define _ASM_TILE_MMU_CONTEXT_H
#include <linux/smp.h>
+#include <linux/mm_types.h>
+
#include <asm/setup.h>
#include <asm/page.h>
#include <asm/pgalloc.h>
#define MAXMEM (_VMALLOC_START - PAGE_OFFSET)
/* We have no pmd or pud since we are strictly a two-level page table */
+#define __ARCH_USE_5LEVEL_HACK
#include <asm-generic/pgtable-nopmd.h>
static inline int pud_huge_page(pud_t pud) { return 0; }
#ifndef __ASSEMBLY__
/* We have no pud since we are a three-level page table. */
+#define __ARCH_USE_5LEVEL_HACK
#include <asm-generic/pgtable-nopud.h>
/*
#include <linux/types.h>
#include <linux/sched.h>
+#include <linux/sched/debug.h>
+
#include <asm/backtrace.h>
#include <asm/page.h>
#include <hv/hypervisor.h>
*/
#include <linux/sched.h>
+#include <linux/sched/task_stack.h>
#include <linux/mm.h>
#include <linux/smp.h>
#include <linux/kernel.h>
#include <linux/kdebug.h>
#include <linux/uaccess.h>
#include <linux/module.h>
+#include <linux/sched/task_stack.h>
+
#include <asm/cacheflush.h>
static tile_bundle_bits singlestep_insn = TILEGX_BPT_BUNDLE | DIE_SSTEPBP;
*/
#include <linux/sched.h>
+#include <linux/sched/debug.h>
+#include <linux/sched/task.h>
+#include <linux/sched/task_stack.h>
#include <linux/preempt.h>
#include <linux/module.h>
#include <linux/fs.h>
#include <linux/elf.h>
#include <linux/tracehook.h>
#include <linux/context_tracking.h>
+#include <linux/sched/task_stack.h>
+
#include <asm/traps.h>
#include <arch/chip.h>
*/
#include <linux/sched.h>
+#include <linux/sched/debug.h>
+#include <linux/sched/task_stack.h>
#include <linux/mm.h>
#include <linux/smp.h>
#include <linux/kernel.h>
#include <linux/init.h>
#include <linux/kernel.h>
#include <linux/mm.h>
-#include <linux/sched.h>
+#include <linux/sched/mm.h>
+#include <linux/sched/task.h>
#include <linux/kernel_stat.h>
#include <linux/bootmem.h>
#include <linux/notifier.h>
*/
#include <linux/sched.h>
+#include <linux/sched/debug.h>
+#include <linux/sched/task_stack.h>
#include <linux/kernel.h>
#include <linux/kprobes.h>
#include <linux/module.h>
#include <linux/clockchips.h>
#include <linux/hardirq.h>
#include <linux/sched.h>
+#include <linux/sched/clock.h>
#include <linux/smp.h>
#include <linux/delay.h>
#include <linux/module.h>
*/
#include <linux/sched.h>
+#include <linux/sched/debug.h>
#include <linux/kernel.h>
#include <linux/kprobes.h>
#include <linux/kdebug.h>
#include <linux/smp.h>
#include <linux/ptrace.h>
#include <linux/slab.h>
+#include <linux/sched/debug.h>
+#include <linux/sched/task.h>
#include <linux/thread_info.h>
#include <linux/uaccess.h>
#include <linux/mman.h>
#include <linux/signal.h>
#include <linux/sched.h>
+#include <linux/sched/debug.h>
+#include <linux/sched/task.h>
+#include <linux/sched/task_stack.h>
#include <linux/kernel.h>
#include <linux/errno.h>
#include <linux/string.h>
#include <linux/init.h>
#include <linux/fs.h>
#include <linux/mm.h>
+#include <linux/sched/mm.h>
#include <linux/hugetlb.h>
#include <linux/pagemap.h>
#include <linux/slab.h>
#include <linux/mm.h>
#include <linux/random.h>
#include <linux/limits.h>
-#include <linux/sched.h>
+#include <linux/sched/signal.h>
+#include <linux/sched/mm.h>
#include <linux/mman.h>
#include <linux/compat.h>
#include <linux/irqreturn.h>
#include <linux/kd.h>
-#include <linux/sched.h>
+#include <linux/sched/signal.h>
#include <linux/slab.h>
+
#include "chan.h"
#include <irq_kern.h>
#include <irq_user.h>
#include <linux/module.h>
#include <linux/notifier.h>
#include <linux/reboot.h>
+#include <linux/sched/debug.h>
#include <linux/proc_fs.h>
#include <linux/slab.h>
#include <linux/syscalls.h>
* This software may be used and distributed according to the terms
* of the GNU General Public License, incorporated herein by reference.
*/
-#include <linux/sched.h>
+#include <linux/sched/signal.h>
#include <linux/module.h>
#include <linux/fs.h>
#include <linux/interrupt.h>
#define __UM_MMU_CONTEXT_H
#include <linux/sched.h>
+#include <linux/mm_types.h>
+
#include <asm/mmu.h>
extern void uml_setup_stubs(struct mm_struct *mm);
#ifndef __UM_PGTABLE_2LEVEL_H
#define __UM_PGTABLE_2LEVEL_H
+#define __ARCH_USE_5LEVEL_HACK
#include <asm-generic/pgtable-nopmd.h>
/* PGDIR_SHIFT determines what a third-level page table entry can map */
#ifndef __UM_PGTABLE_3LEVEL_H
#define __UM_PGTABLE_3LEVEL_H
+#define __ARCH_USE_5LEVEL_HACK
#include <asm-generic/pgtable-nopud.h>
/* PGDIR_SHIFT determines what a third-level page table entry can map */
#include <linux/module.h>
#include <linux/fs.h>
#include <linux/ptrace.h>
-#include <linux/sched.h>
+#include <linux/sched/mm.h>
+#include <linux/sched/task.h>
+#include <linux/sched/task_stack.h>
#include <linux/slab.h>
#include <asm/current.h>
#include <asm/processor.h>
#include <linux/random.h>
#include <linux/slab.h>
#include <linux/sched.h>
+#include <linux/sched/debug.h>
+#include <linux/sched/task.h>
+#include <linux/sched/task_stack.h>
#include <linux/seq_file.h>
#include <linux/tick.h>
#include <linux/threads.h>
* Licensed under the GPL
*/
-#include <linux/sched.h>
+#include <linux/sched/signal.h>
+#include <linux/sched/task.h>
+#include <linux/sched/mm.h>
#include <linux/spinlock.h>
#include <linux/slab.h>
#include <linux/oom.h>
*/
#include <linux/mm.h>
-#include <linux/sched.h>
+#include <linux/sched/signal.h>
#include <linux/slab.h>
+
#include <asm/pgalloc.h>
#include <asm/pgtable.h>
#include <asm/sections.h>
*/
#include <linux/init.h>
-#include <linux/sched.h>
+#include <linux/sched/mm.h>
+#include <linux/sched/task_stack.h>
+#include <linux/sched/task.h>
+
#include <as-layout.h>
#include <kern.h>
#include <os.h>
#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/sched.h>
+#include <linux/sched/debug.h>
+#include <linux/sched/task_stack.h>
+
#include <asm/sysrq.h>
#include <asm/stacktrace.h>
#include <os.h>
#include <linux/mm.h>
#include <linux/module.h>
-#include <linux/sched.h>
+#include <linux/sched/signal.h>
+
#include <asm/pgtable.h>
#include <asm/tlbflush.h>
#include <as-layout.h>
*/
#include <linux/mm.h>
-#include <linux/sched.h>
+#include <linux/sched/signal.h>
#include <linux/hardirq.h>
#include <linux/module.h>
#include <linux/uaccess.h>
+#include <linux/sched/debug.h>
#include <asm/current.h>
#include <asm/pgtable.h>
#include <asm/tlbflush.h>
#include <linux/string.h>
#include <linux/utsname.h>
#include <linux/sched.h>
+#include <linux/sched/task.h>
#include <linux/kmsg_dump.h>
+
#include <asm/pgtable.h>
#include <asm/processor.h>
#include <asm/sections.h>
#ifndef __UNICORE_PGTABLE_H__
#define __UNICORE_PGTABLE_H__
+#define __ARCH_USE_5LEVEL_HACK
#include <asm-generic/pgtable-nopmd.h>
#include <asm/cpu-single.h>
#include <linux/types.h>
#include <linux/kernel.h>
#include <linux/signal.h>
-#include <linux/sched.h>
+#include <linux/sched/signal.h>
#include <linux/init.h>
#include <asm/fpu-ucf64.h>
#include <linux/module.h>
#include <linux/sched.h>
+#include <linux/sched/debug.h>
+#include <linux/sched/task.h>
+#include <linux/sched/task_stack.h>
#include <linux/kernel.h>
#include <linux/mm.h>
#include <linux/stddef.h>
#include <linux/ptrace.h>
#include <linux/signal.h>
#include <linux/uaccess.h>
+#include <linux/sched/task_stack.h>
/*
* this routine will get a word off of the processes privileged stack.
*/
#include <linux/module.h>
#include <linux/sched.h>
+#include <linux/sched/debug.h>
#include <linux/stacktrace.h>
#include <asm/stacktrace.h>
*/
#include <linux/module.h>
#include <linux/signal.h>
+#include <linux/sched/signal.h>
+#include <linux/sched/debug.h>
+#include <linux/sched/task_stack.h>
#include <linux/spinlock.h>
#include <linux/personality.h>
#include <linux/kallsyms.h>
*/
#include <linux/compiler.h>
#include <linux/kernel.h>
+#include <linux/sched/debug.h>
#include <linux/errno.h>
#include <linux/string.h>
#include <linux/init.h>
#include <linux/kprobes.h>
#include <linux/uaccess.h>
#include <linux/page-flags.h>
-#include <linux/sched.h>
+#include <linux/sched/signal.h>
#include <linux/io.h>
#include <asm/pgtable.h>
CONFIG_SKY2=y
CONFIG_FORCEDETH=y
CONFIG_8139TOO=y
+CONFIG_R8169=y
CONFIG_FDDI=y
CONFIG_INPUT_POLLDEV=y
# CONFIG_INPUT_MOUSEDEV_PSAUX is not set
#include <linux/kernel.h>
#include <linux/sched.h>
+#include <linux/sched/task_stack.h>
#include <linux/mm.h>
#include <linux/smp.h>
#include <linux/errno.h>
380 i386 pkey_mprotect sys_pkey_mprotect
381 i386 pkey_alloc sys_pkey_alloc
382 i386 pkey_free sys_pkey_free
+383 i386 statx sys_statx
329 common pkey_mprotect sys_pkey_mprotect
330 common pkey_alloc sys_pkey_alloc
331 common pkey_free sys_pkey_free
+332 common statx sys_statx
#
# x32-specific system call numbers start at 512 to avoid cache impact
#include <linux/mm.h>
#include <linux/err.h>
#include <linux/sched.h>
+#include <linux/sched/task_stack.h>
#include <linux/slab.h>
#include <linux/init.h>
#include <linux/random.h>
#include <linux/kernel.h>
#include <linux/timer.h>
+#include <linux/sched/signal.h>
+#include <linux/mm_types.h>
#include <linux/syscalls.h>
#include <linux/ratelimit.h>
return &amd_f15_PMC20;
}
case AMD_EVENT_NB:
- /* moved to perf_event_amd_uncore.c */
+ /* moved to uncore.c */
return &emptyconstraint;
default:
return &emptyconstraint;
#include <linux/pci.h>
#include <linux/ptrace.h>
#include <linux/syscore_ops.h>
+#include <linux/sched/clock.h>
#include <asm/apic.h>
#include <linux/export.h>
#include <linux/init.h>
#include <linux/kdebug.h>
-#include <linux/sched.h>
+#include <linux/sched/mm.h>
+#include <linux/sched/clock.h>
#include <linux/uaccess.h>
#include <linux/slab.h>
#include <linux/cpu.h>
/*
- * perf_event_intel_cstate.c: support cstate residency counters
+ * Support cstate residency counters
*
* Copyright (C) 2015, Intel Corp.
* Author: Kan Liang (kan.liang@intel.com)
/*
- * perf_event_intel_rapl.c: support Intel RAPL energy consumption counters
+ * Support Intel RAPL energy consumption counters
* Copyright (C) 2013 Google, Inc., Stephane Eranian
*
* Intel RAPL interface is specified in the IA-32 Manual Vol3b
extern struct pci_extra_dev *uncore_extra_pci_dev;
extern struct event_constraint uncore_constraint_empty;
-/* perf_event_intel_uncore_snb.c */
+/* uncore_snb.c */
int snb_uncore_pci_init(void);
int ivb_uncore_pci_init(void);
int hsw_uncore_pci_init(void);
void skl_uncore_cpu_init(void);
int snb_pci2phy_map_init(int devid);
-/* perf_event_intel_uncore_snbep.c */
+/* uncore_snbep.c */
int snbep_uncore_pci_init(void);
void snbep_uncore_cpu_init(void);
int ivbep_uncore_pci_init(void);
int skx_uncore_pci_init(void);
void skx_uncore_cpu_init(void);
-/* perf_event_intel_uncore_nhmex.c */
+/* uncore_nhmex.c */
void nhmex_uncore_cpu_init(void);
clocksource_register_hz(&hyperv_cs_tsc, NSEC_PER_SEC/100);
return;
}
+register_msr_cs:
#endif
/*
* For 32 bit guests just use the MSR based mechanism for reading
* the partition counter.
*/
-register_msr_cs:
hyperv_cs = &hyperv_cs_msr;
if (ms_hyperv.features & HV_X64_MSR_TIME_REF_COUNT_AVAILABLE)
clocksource_register_hz(&hyperv_cs_msr, NSEC_PER_SEC/100);
#include <linux/init.h>
#include <linux/jiffies.h>
#include <linux/perf_event.h>
+#include <linux/sched/task_stack.h>
#include <linux/uaccess.h>
#include <asm/pgalloc.h>
*/
#include <linux/sched.h>
+#include <linux/sched/task_stack.h>
#include <linux/mm.h>
#include <linux/smp.h>
#include <linux/kernel.h>
#include <linux/user.h>
#include <linux/elfcore.h>
+#include <linux/mm_types.h>
+
#include <asm/debugreg.h>
/*
#define _ASM_X86_APIC_H
#include <linux/cpumask.h>
-#include <linux/pm.h>
#include <asm/alternative.h>
#include <asm/cpufeature.h>
#define X86_FEATURE_PKU (16*32+ 3) /* Protection Keys for Userspace */
#define X86_FEATURE_OSPKE (16*32+ 4) /* OS Protection Keys Enable */
#define X86_FEATURE_AVX512_VPOPCNTDQ (16*32+14) /* POPCNT for vectors of DW/QW */
-#define X86_FEATURE_RDPID (16*32+ 22) /* RDPID instruction */
+#define X86_FEATURE_LA57 (16*32+16) /* 5-level page tables */
+#define X86_FEATURE_RDPID (16*32+22) /* RDPID instruction */
/* AMD-defined CPU features, CPUID level 0x80000007 (ebx), word 17 */
#define X86_FEATURE_OVERFLOW_RECOV (17*32+0) /* MCA overflow recovery support */
asm volatile("ltr %w0"::"q" (GDT_ENTRY_TSS*8));
}
+DECLARE_PER_CPU(bool, __tss_limit_invalid);
+
static inline void force_reload_TR(void)
{
struct desc_struct *d = get_cpu_gdt_table(smp_processor_id());
write_gdt_entry(d, GDT_ENTRY_TSS, &tss, DESC_TSS);
load_TR_desc();
+ this_cpu_write(__tss_limit_invalid, false);
}
-DECLARE_PER_CPU(bool, need_tr_refresh);
-
-static inline void refresh_TR(void)
+/*
+ * Call this if you need the TSS limit to be correct, which should be the case
+ * if and only if you have TIF_IO_BITMAP set or you're switching to a task
+ * with TIF_IO_BITMAP set.
+ */
+static inline void refresh_tss_limit(void)
{
DEBUG_LOCKS_WARN_ON(preemptible());
- if (unlikely(this_cpu_read(need_tr_refresh))) {
+ if (unlikely(this_cpu_read(__tss_limit_invalid)))
force_reload_TR();
- this_cpu_write(need_tr_refresh, false);
- }
}
/*
if (unlikely(test_thread_flag(TIF_IO_BITMAP)))
force_reload_TR();
else
- this_cpu_write(need_tr_refresh, true);
+ this_cpu_write(__tss_limit_invalid, true);
}
static inline void native_load_gdt(const struct desc_ptr *dtr)
#define INTEL_FAM6_ATOM_MERRIFIELD 0x4A /* Tangier */
#define INTEL_FAM6_ATOM_MOOREFIELD 0x5A /* Anniedale */
#define INTEL_FAM6_ATOM_GOLDMONT 0x5C
+#define INTEL_FAM6_ATOM_GEMINI_LAKE 0x7A
#define INTEL_FAM6_ATOM_DENVERTON 0x5F /* Goldmont Microserver */
/* Xeon Phi */
#ifdef CONFIG_INTEL_RDT_A
+#include <linux/sched.h>
#include <linux/kernfs.h>
#include <linux/jump_label.h>
#define _ASM_X86_MPX_H
#include <linux/types.h>
+#include <linux/mm_types.h>
+
#include <asm/ptrace.h>
#include <asm/insn.h>
#define MSR_FSB_FREQ 0x000000cd
#define MSR_PLATFORM_INFO 0x000000ce
-#define MSR_NHM_SNB_PKG_CST_CFG_CTL 0x000000e2
+#define MSR_PKG_CST_CONFIG_CONTROL 0x000000e2
#define NHM_C3_AUTO_DEMOTE (1UL << 25)
#define NHM_C1_AUTO_DEMOTE (1UL << 26)
#define ATM_LNC_C6_AUTO_DEMOTE (1UL << 25)
/* C-state Residency Counters */
#define MSR_PKG_C3_RESIDENCY 0x000003f8
#define MSR_PKG_C6_RESIDENCY 0x000003f9
+#define MSR_ATOM_PKG_C6_RESIDENCY 0x000003fa
#define MSR_PKG_C7_RESIDENCY 0x000003fa
#define MSR_CORE_C3_RESIDENCY 0x000003fc
#define MSR_CORE_C6_RESIDENCY 0x000003fd
#define MSR_PKG_BOTH_CORE_GFXE_C0_RES 0x0000065B
#define MSR_CORE_C1_RES 0x00000660
+#define MSR_MODULE_C6_RES_MS 0x00000664
#define MSR_CC6_DEMOTION_POLICY_CONFIG 0x00000668
#define MSR_MC6_DEMOTION_POLICY_CONFIG 0x00000669
+#define MSR_ATOM_CORE_RATIOS 0x0000066a
+#define MSR_ATOM_CORE_VIDS 0x0000066b
+#define MSR_ATOM_CORE_TURBO_RATIOS 0x0000066c
+#define MSR_ATOM_CORE_TURBO_VIDS 0x0000066d
+
+
#define MSR_CORE_PERF_LIMIT_REASONS 0x00000690
#define MSR_GFX_PERF_LIMIT_REASONS 0x000006B0
#define MSR_RING_PERF_LIMIT_REASONS 0x000006B1
#define MSR_IA32_TEMPERATURE_TARGET 0x000001a2
+#define MSR_MISC_FEATURE_CONTROL 0x000001a4
#define MSR_MISC_PWR_MGMT 0x000001aa
#define MSR_IA32_ENERGY_PERF_BIAS 0x000001b0
#define _ASM_X86_MWAIT_H
#include <linux/sched.h>
+#include <linux/sched/idle.h>
#include <asm/cpufeature.h>
}
#if CONFIG_PGTABLE_LEVELS > 3
+#include <asm-generic/5level-fixup.h>
+
typedef struct { pudval_t pud; } pud_t;
static inline pud_t native_make_pud(pmdval_t val)
return pud.pud;
}
#else
+#define __ARCH_USE_5LEVEL_HACK
#include <asm-generic/pgtable-nopud.h>
static inline pudval_t native_pud_val(pud_t pud)
return pmd.pmd;
}
#else
+#define __ARCH_USE_5LEVEL_HACK
#include <asm-generic/pgtable-nopmd.h>
static inline pmdval_t native_pmd_val(pmd_t pmd)
static inline
bool mm_pkey_is_allocated(struct mm_struct *mm, int pkey)
{
+ /*
+ * "Allocated" pkeys are those that have been returned
+ * from pkey_alloc(). pkey 0 is special, and never
+ * returned from pkey_alloc().
+ */
+ if (pkey <= 0)
+ return false;
+ if (pkey >= arch_max_pkey())
+ return false;
return mm_pkey_allocation_map(mm) & (1U << pkey);
}
static inline
int mm_pkey_free(struct mm_struct *mm, int pkey)
{
- /*
- * pkey 0 is special, always allocated and can never
- * be freed.
- */
- if (!pkey)
- return -EINVAL;
if (!mm_pkey_is_allocated(mm, pkey))
return -EINVAL;
--- /dev/null
+#ifndef _ASM_X86_PURGATORY_H
+#define _ASM_X86_PURGATORY_H
+
+#ifndef __ASSEMBLY__
+#include <linux/purgatory.h>
+
+extern void purgatory(void);
+/*
+ * These forward declarations serve two purposes:
+ *
+ * 1) Make sparse happy when checking arch/purgatory
+ * 2) Document that these are required to be global so the symbol
+ * lookup in kexec works
+ */
+extern unsigned long purgatory_backup_dest;
+extern unsigned long purgatory_backup_src;
+extern unsigned long purgatory_backup_sz;
+#endif /* __ASSEMBLY__ */
+
+#endif /* _ASM_PURGATORY_H */
static inline void __flush_tlb_all(void)
{
- if (static_cpu_has(X86_FEATURE_PGE))
+ if (boot_cpu_has(X86_FEATURE_PGE))
__flush_tlb_global();
else
__flush_tlb();
__u32 header;
__u16 version;
__u32 realmode_swtch;
- __u16 start_sys;
+ __u16 start_sys_seg;
__u16 kernel_version;
__u8 type_of_loader;
__u8 loadflags;
#include <linux/init.h>
#include <linux/mm.h>
#include <linux/sched.h>
+#include <linux/sched/debug.h>
#include <linux/string.h>
#include <linux/spinlock.h>
#include <linux/pci.h>
static inline void __x2apic_enable(void) { }
#endif /* !CONFIG_X86_X2APIC */
-static int __init try_to_enable_IR(void)
-{
-#ifdef CONFIG_X86_IO_APIC
- if (!x2apic_enabled() && skip_ioapic_setup) {
- pr_info("Not enabling interrupt remapping due to skipped IO-APIC setup\n");
- return -1;
- }
-#endif
- return irq_remapping_enable();
-}
-
void __init enable_IR_x2apic(void)
{
unsigned long flags;
int ret, ir_stat;
- if (skip_ioapic_setup)
+ if (skip_ioapic_setup) {
+ pr_info("Not enabling interrupt remapping due to skipped IO-APIC setup\n");
return;
+ }
ir_stat = irq_remapping_prepare();
if (ir_stat < 0 && !x2apic_supported())
/* If irq_remapping_prepare() succeeded, try to enable it */
if (ir_stat >= 0)
- ir_stat = try_to_enable_IR();
+ ir_stat = irq_remapping_enable();
/* ir_stat contains the remap mode or an error code */
try_to_enable_x2apic(ir_stat);
/* Allocate a new cpuid. */
if (nr_logical_cpuids >= nr_cpu_ids) {
- WARN_ONCE(1, "Only %d processors supported."
+ WARN_ONCE(1, "APIC: NR_CPUS/possible_cpus limit of %i reached. "
"Processor %d/0x%x and the rest are ignored.\n",
- nr_cpu_ids - 1, nr_logical_cpuids, apicid);
- return -1;
+ nr_cpu_ids, nr_logical_cpuids, apicid);
+ return -EINVAL;
}
cpuid_to_apicid[nr_logical_cpuids] = apicid;
#include <linux/apm_bios.h>
#include <linux/init.h>
#include <linux/time.h>
-#include <linux/sched.h>
+#include <linux/sched/signal.h>
+#include <linux/sched/cputime.h>
#include <linux/pm.h>
#include <linux/capability.h>
#include <linux/device.h>
#include <linux/io.h>
#include <linux/sched.h>
+#include <linux/sched/clock.h>
#include <linux/random.h>
#include <asm/processor.h>
#include <asm/apic.h>
if (c->x86_power & (1 << 8)) {
set_cpu_cap(c, X86_FEATURE_CONSTANT_TSC);
set_cpu_cap(c, X86_FEATURE_NONSTOP_TSC);
- if (check_tsc_unstable())
- clear_sched_clock_stable();
- } else {
- clear_sched_clock_stable();
}
/* Bit 12 of 8000_0007 edx is accumulated power mechanism. */
#include <linux/sched.h>
+#include <linux/sched/clock.h>
#include <asm/cpufeature.h>
#include <asm/e820.h>
#ifdef CONFIG_X86_64
set_cpu_cap(c, X86_FEATURE_SYSENTER32);
#endif
-
- clear_sched_clock_stable();
}
static void init_centaur(struct cpuinfo_x86 *c)
#include <linux/string.h>
#include <linux/ctype.h>
#include <linux/delay.h>
-#include <linux/sched.h>
+#include <linux/sched/mm.h>
+#include <linux/sched/clock.h>
+#include <linux/sched/task.h>
#include <linux/init.h>
#include <linux/kprobes.h>
#include <linux/kgdb.h>
strcpy(c->x86_model_id, "386");
}
#endif
- clear_sched_clock_stable();
}
static const struct cpu_dev default_cpu = {
*/
if (this_cpu->c_init)
this_cpu->c_init(c);
- else
- clear_sched_clock_stable();
/* Disable the PN if appropriate */
squash_the_stupid_serial_number(c);
#include <asm/tsc.h>
#include <asm/cpufeature.h>
#include <linux/sched.h>
+#include <linux/sched/clock.h>
#include "cpu.h"
set_cpu_cap(c, X86_FEATURE_CYRIX_ARR);
break;
}
- clear_sched_clock_stable();
}
static void init_cyrix(struct cpuinfo_x86 *c)
#include <linux/bitops.h>
#include <linux/smp.h>
#include <linux/sched.h>
+#include <linux/sched/clock.h>
#include <linux/thread_info.h>
#include <linux/init.h>
#include <linux/uaccess.h>
if (c->x86_power & (1 << 8)) {
set_cpu_cap(c, X86_FEATURE_CONSTANT_TSC);
set_cpu_cap(c, X86_FEATURE_NONSTOP_TSC);
- if (check_tsc_unstable())
- clear_sched_clock_stable();
- } else {
- clear_sched_clock_stable();
}
/* Penwell and Cloverview have the TSC which doesn't sleep on S3 */
#include <linux/cacheinfo.h>
#include <linux/cpu.h>
#include <linux/sched.h>
+#include <linux/capability.h>
#include <linux/sysfs.h>
#include <linux/pci.h>
#include <linux/sysfs.h>
#include <linux/kernfs.h>
#include <linux/seq_file.h>
-#include <linux/sched.h>
+#include <linux/sched/signal.h>
+#include <linux/sched/task.h>
#include <linux/slab.h>
-#include <linux/cpu.h>
#include <linux/task_work.h>
#include <uapi/linux/magic.h>
#include <linux/kernel.h>
#include <linux/sched.h>
+#include <linux/sched/clock.h>
#include <linux/mm.h>
#include <asm/cpufeature.h>
#include <asm/msr.h>
if (xlvl >= 0x80860001)
c->x86_capability[CPUID_8086_0001_EDX] = cpuid_edx(0x80860001);
}
-
- clear_sched_clock_stable();
}
static void init_transmeta(struct cpuinfo_x86 *c)
#include <asm/hypervisor.h>
#include <asm/timer.h>
#include <asm/apic.h>
-#include <asm/timer.h>
#undef pr_fmt
#define pr_fmt(fmt) "vmware: " fmt
#include <linux/mm.h>
#include <linux/sched.h>
+#include <linux/sched/debug.h>
#include <linux/init_task.h>
#include <linux/fs.h>
#include <linux/kdebug.h>
#include <linux/module.h>
#include <linux/ptrace.h>
+#include <linux/sched/debug.h>
+#include <linux/sched/task_stack.h>
#include <linux/ftrace.h>
#include <linux/kexec.h>
#include <linux/bug.h>
* Copyright (C) 1991, 1992 Linus Torvalds
* Copyright (C) 2000, 2001, 2002 Andi Kleen, SuSE Labs
*/
+#include <linux/sched/debug.h>
#include <linux/kallsyms.h>
#include <linux/kprobes.h>
#include <linux/uaccess.h>
* Copyright (C) 1991, 1992 Linus Torvalds
* Copyright (C) 2000, 2001, 2002 Andi Kleen, SuSE Labs
*/
+#include <linux/sched/debug.h>
#include <linux/kallsyms.h>
#include <linux/kprobes.h>
#include <linux/uaccess.h>
#include <asm/cmdline.h>
#include <linux/sched.h>
+#include <linux/sched/task.h>
#include <linux/init.h>
/*
#include <asm/fpu/signal.h>
#include <asm/fpu/regset.h>
#include <asm/fpu/xstate.h>
+#include <linux/sched/task_stack.h>
/*
* The xstateregs_active() routine is the same as the regset_fpregs_active() routine,
{
int enable_irqs = irqs_disabled();
- /* We may be called with interrupts disbled (on bootup). */
+ /* We may be called with interrupts disabled (on bootup). */
if (enable_irqs)
local_irq_enable();
on_each_cpu(do_sync_core, NULL, 1);
irq_domain_deactivate_irq(irq_get_irq_data(hdev->irq));
irq_domain_activate_irq(irq_get_irq_data(hdev->irq));
- disable_irq(hdev->irq);
+ disable_hardirq(hdev->irq);
irq_set_affinity(hdev->irq, cpumask_of(hdev->cpu));
enable_irq(hdev->irq);
}
*/
#include <linux/sched.h>
+#include <linux/sched/task_stack.h>
#include <linux/kernel.h>
#include <linux/capability.h>
#include <linux/errno.h>
t->io_bitmap_ptr = bitmap;
set_thread_flag(TIF_IO_BITMAP);
+ /*
+ * Now that we have an IO bitmap, we need our TSS limit to be
+ * correct. It's fine if we are preempted after doing this:
+ * with TIF_IO_BITMAP set, context switches will keep our TSS
+ * limit correct.
+ */
preempt_disable();
- refresh_TR();
+ refresh_tss_limit();
preempt_enable();
}
#include <linux/ftrace.h>
#include <linux/uaccess.h>
#include <linux/smp.h>
+#include <linux/sched/task_stack.h>
#include <asm/io_apic.h>
#include <asm/apic.h>
struct dentry *dbp, *version, *data;
int error = -ENOMEM;
- dbp = debugfs_create_dir("boot_params", NULL);
+ dbp = debugfs_create_dir("boot_params", arch_debugfs_dir);
if (!dbp)
return -ENOMEM;
#endif
/* Ensure if the instruction can be boostable */
-extern int can_boost(kprobe_opcode_t *instruction);
+extern int can_boost(kprobe_opcode_t *instruction, void *addr);
/* Recover instruction if given address is probed */
extern unsigned long recover_probed_instruction(kprobe_opcode_t *buf,
unsigned long addr);
#include <linux/slab.h>
#include <linux/hardirq.h>
#include <linux/preempt.h>
+#include <linux/sched/debug.h>
#include <linux/extable.h>
#include <linux/kdebug.h>
#include <linux/kallsyms.h>
* Returns non-zero if opcode is boostable.
* RIP relative instructions are adjusted at copying time in 64 bits mode
*/
-int can_boost(kprobe_opcode_t *opcodes)
+int can_boost(kprobe_opcode_t *opcodes, void *addr)
{
kprobe_opcode_t opcode;
kprobe_opcode_t *orig_opcodes = opcodes;
- if (search_exception_tables((unsigned long)opcodes))
+ if (search_exception_tables((unsigned long)addr))
return 0; /* Page fault may occur on this address. */
retry:
* __copy_instruction can modify the displacement of the instruction,
* but it doesn't affect boostable check.
*/
- if (can_boost(p->ainsn.insn))
+ if (can_boost(p->ainsn.insn, p->addr))
p->ainsn.boostable = 0;
else
p->ainsn.boostable = -1;
while (len < RELATIVEJUMP_SIZE) {
ret = __copy_instruction(dest + len, src + len);
- if (!ret || !can_boost(dest + len))
+ if (!ret || !can_boost(dest + len, src + len))
return -EINVAL;
len += ret;
}
#include <linux/hardirq.h>
#include <linux/memblock.h>
#include <linux/sched.h>
+#include <linux/sched/clock.h>
#include <asm/x86_init.h>
#include <asm/reboot.h>
/* Setup copying of backup region */
if (image->type == KEXEC_TYPE_CRASH) {
- ret = kexec_purgatory_get_set_symbol(image, "backup_dest",
+ ret = kexec_purgatory_get_set_symbol(image,
+ "purgatory_backup_dest",
&image->arch.backup_load_addr,
sizeof(image->arch.backup_load_addr), 0);
if (ret)
return ret;
- ret = kexec_purgatory_get_set_symbol(image, "backup_src",
+ ret = kexec_purgatory_get_set_symbol(image,
+ "purgatory_backup_src",
&image->arch.backup_src_start,
sizeof(image->arch.backup_src_start), 0);
if (ret)
return ret;
- ret = kexec_purgatory_get_set_symbol(image, "backup_sz",
+ ret = kexec_purgatory_get_set_symbol(image,
+ "purgatory_backup_sz",
&image->arch.backup_src_sz,
sizeof(image->arch.backup_src_sz), 0);
if (ret)
#include <linux/spinlock.h>
#include <linux/kprobes.h>
#include <linux/kdebug.h>
+#include <linux/sched/debug.h>
#include <linux/nmi.h>
#include <linux/debugfs.h>
#include <linux/delay.h>
#include <linux/ratelimit.h>
#include <linux/slab.h>
#include <linux/export.h>
+#include <linux/sched/clock.h>
#if defined(CONFIG_EDAC)
#include <linux/edac.h>
#include <linux/errno.h>
#include <linux/kernel.h>
#include <linux/sched.h>
+#include <linux/sched/task_stack.h>
#include <linux/perf_event.h>
#include <linux/bug.h>
#include <linux/stddef.h>
#include <linux/prctl.h>
#include <linux/slab.h>
#include <linux/sched.h>
+#include <linux/sched/idle.h>
+#include <linux/sched/debug.h>
+#include <linux/sched/task.h>
+#include <linux/sched/task_stack.h>
#include <linux/init.h>
#include <linux/export.h>
#include <linux/pm.h>
};
EXPORT_PER_CPU_SYMBOL(cpu_tss);
-DEFINE_PER_CPU(bool, need_tr_refresh);
-EXPORT_PER_CPU_SYMBOL_GPL(need_tr_refresh);
+DEFINE_PER_CPU(bool, __tss_limit_invalid);
+EXPORT_PER_CPU_SYMBOL_GPL(__tss_limit_invalid);
/*
* this gets called so that we can store lazy state into memory and copy the
* Make sure that the TSS limit is correct for the CPU
* to notice the IO bitmap.
*/
- refresh_TR();
+ refresh_tss_limit();
} else if (test_tsk_thread_flag(prev_p, TIF_IO_BITMAP)) {
/*
* Clear any possible leftover bits:
#include <linux/cpu.h>
#include <linux/errno.h>
#include <linux/sched.h>
+#include <linux/sched/task.h>
+#include <linux/sched/task_stack.h>
#include <linux/fs.h>
#include <linux/kernel.h>
#include <linux/mm.h>
#include <linux/cpu.h>
#include <linux/errno.h>
#include <linux/sched.h>
+#include <linux/sched/task.h>
+#include <linux/sched/task_stack.h>
#include <linux/fs.h>
#include <linux/kernel.h>
#include <linux/mm.h>
#include <linux/kernel.h>
#include <linux/sched.h>
+#include <linux/sched/task_stack.h>
#include <linux/mm.h>
#include <linux/smp.h>
#include <linux/errno.h>
#include <linux/sched.h>
#include <linux/gfp.h>
#include <linux/bootmem.h>
+#include <linux/nmi.h>
+
#include <asm/fixmap.h>
#include <asm/pvclock.h>
DMI_MATCH(DMI_BOARD_NAME, "P4S800"),
},
},
+ { /* Handle problems with rebooting on ASUS EeeBook X205TA */
+ .callback = set_acpi_reboot,
+ .ident = "ASUS EeeBook X205TA",
+ .matches = {
+ DMI_MATCH(DMI_SYS_VENDOR, "ASUSTeK COMPUTER INC."),
+ DMI_MATCH(DMI_PRODUCT_NAME, "X205TA"),
+ },
+ },
+ { /* Handle problems with rebooting on ASUS EeeBook X205TAW */
+ .callback = set_acpi_reboot,
+ .ident = "ASUS EeeBook X205TAW",
+ .matches = {
+ DMI_MATCH(DMI_SYS_VENDOR, "ASUSTeK COMPUTER INC."),
+ DMI_MATCH(DMI_PRODUCT_NAME, "X205TAW"),
+ },
+ },
/* Certec */
{ /* Handle problems with rebooting on Certec BPC600 */
#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
#include <linux/sched.h>
+#include <linux/sched/task_stack.h>
#include <linux/mm.h>
#include <linux/smp.h>
#include <linux/kernel.h>
#include <linux/smp.h>
#include <linux/export.h>
#include <linux/sched.h>
+#include <linux/sched/topology.h>
+#include <linux/sched/hotplug.h>
+#include <linux/sched/task_stack.h>
#include <linux/percpu.h>
#include <linux/bootmem.h>
#include <linux/err.h>
* Copyright (C) 2006-2009 Red Hat, Inc., Ingo Molnar <mingo@redhat.com>
*/
#include <linux/sched.h>
+#include <linux/sched/debug.h>
+#include <linux/sched/task_stack.h>
#include <linux/stacktrace.h>
#include <linux/export.h>
#include <linux/uaccess.h>
* x86 single-step support code, common to 32-bit and 64-bit.
*/
#include <linux/sched.h>
+#include <linux/sched/task_stack.h>
#include <linux/mm.h>
#include <linux/ptrace.h>
#include <asm/desc.h>
#include <linux/errno.h>
#include <linux/sched.h>
+#include <linux/sched/mm.h>
#include <linux/syscalls.h>
#include <linux/mm.h>
#include <linux/fs.h>
#include <linux/errno.h>
#include <linux/kexec.h>
#include <linux/sched.h>
+#include <linux/sched/task_stack.h>
#include <linux/timer.h>
#include <linux/init.h>
#include <linux/bug.h>
#include <linux/kernel.h>
#include <linux/sched.h>
+#include <linux/sched/clock.h>
#include <linux/init.h>
#include <linux/export.h>
#include <linux/timer.h>
{
return paravirt_sched_clock();
}
+
+static inline bool using_native_sched_clock(void)
+{
+ return pv_time_ops.sched_clock == native_sched_clock;
+}
#else
unsigned long long
sched_clock(void) __attribute__((alias("native_sched_clock")));
+
+static inline bool using_native_sched_clock(void) { return true; }
#endif
int check_tsc_unstable(void)
{
if (tsc_unstable)
return;
+
tsc_unstable = 1;
- clear_sched_clock_stable();
+ if (using_native_sched_clock())
+ clear_sched_clock_stable();
disable_sched_clock_irqtime();
pr_info("Marking TSC unstable due to clocksource watchdog\n");
}
void mark_tsc_unstable(char *reason)
{
- if (!tsc_unstable) {
- tsc_unstable = 1;
+ if (tsc_unstable)
+ return;
+
+ tsc_unstable = 1;
+ if (using_native_sched_clock())
clear_sched_clock_stable();
- disable_sched_clock_irqtime();
- pr_info("Marking TSC unstable due to %s\n", reason);
- /* Change only the rating, when not registered */
- if (clocksource_tsc.mult)
- clocksource_mark_unstable(&clocksource_tsc);
- else {
- clocksource_tsc.flags |= CLOCK_SOURCE_UNSTABLE;
- clocksource_tsc.rating = 0;
- }
+ disable_sched_clock_irqtime();
+ pr_info("Marking TSC unstable due to %s\n", reason);
+ /* Change only the rating, when not registered */
+ if (clocksource_tsc.mult) {
+ clocksource_mark_unstable(&clocksource_tsc);
+ } else {
+ clocksource_tsc.flags |= CLOCK_SOURCE_UNSTABLE;
+ clocksource_tsc.rating = 0;
}
}
#include <linux/sched.h>
+#include <linux/sched/task.h>
+#include <linux/sched/task_stack.h>
#include <asm/ptrace.h>
#include <asm/bitops.h>
#include <asm/stacktrace.h>
#include <linux/interrupt.h>
#include <linux/syscalls.h>
#include <linux/sched.h>
+#include <linux/sched/task_stack.h>
#include <linux/kernel.h>
#include <linux/signal.h>
#include <linux/string.h>
#include <linux/export.h>
#include <linux/vmalloc.h>
#include <linux/uaccess.h>
+#include <linux/sched/stat.h>
+
#include <asm/processor.h>
#include <asm/user.h>
#include <asm/fpu/xstate.h>
#include <linux/kvm_host.h>
#include <linux/highmem.h>
+#include <linux/sched/cputime.h>
+
#include <asm/apicdef.h>
#include <trace/events/kvm.h>
#include <linux/kvm_host.h>
#include <linux/slab.h>
#include <linux/export.h>
+#include <linux/rculist.h>
+
#include <trace/events/kvm.h>
#include <asm/msidef.h>
#include <linux/compiler.h>
#include <linux/srcu.h>
#include <linux/slab.h>
+#include <linux/sched/signal.h>
#include <linux/uaccess.h>
#include <linux/hash.h>
#include <linux/kern_levels.h>
*/
#include <linux/kvm_host.h>
+#include <linux/rculist.h>
+
#include <asm/kvm_host.h>
#include <asm/kvm_page_track.h>
.config = config,
};
+ attr.sample_period = (-pmc->counter) & pmc_bitmask(pmc);
+
if (in_tx)
attr.config |= HSW_IN_TX;
- if (in_tx_cp)
+ if (in_tx_cp) {
+ /*
+ * HSW_IN_TX_CHECKPOINTED is not supported with nonzero
+ * period. Just clear the sample period so at least
+ * allocating the counter doesn't fail.
+ */
+ attr.sample_period = 0;
attr.config |= HSW_IN_TX_CHECKPOINTED;
-
- attr.sample_period = (-pmc->counter) & pmc_bitmask(pmc);
+ }
event = perf_event_create_kernel_counter(&attr, -1, current,
intr ? kvm_perf_overflow_intr :
static unsigned long segment_base(u16 selector)
{
struct desc_ptr *gdt = this_cpu_ptr(&host_gdt);
- struct desc_struct *d;
struct desc_struct *table;
unsigned long v;
static int handle_vmclear(struct kvm_vcpu *vcpu)
{
struct vcpu_vmx *vmx = to_vmx(vcpu);
+ u32 zero = 0;
gpa_t vmptr;
- struct vmcs12 *vmcs12;
- struct page *page;
if (!nested_vmx_check_permission(vcpu))
return 1;
if (vmptr == vmx->nested.current_vmptr)
nested_release_vmcs12(vmx);
- page = nested_get_page(vcpu, vmptr);
- if (page == NULL) {
- /*
- * For accurate processor emulation, VMCLEAR beyond available
- * physical memory should do nothing at all. However, it is
- * possible that a nested vmx bug, not a guest hypervisor bug,
- * resulted in this case, so let's shut down before doing any
- * more damage:
- */
- kvm_make_request(KVM_REQ_TRIPLE_FAULT, vcpu);
- return 1;
- }
- vmcs12 = kmap(page);
- vmcs12->launch_state = 0;
- kunmap(page);
- nested_release_page(page);
+ kvm_vcpu_write_guest(vcpu,
+ vmptr + offsetof(struct vmcs12, launch_state),
+ &zero, sizeof(zero));
nested_free_vmcs02(vmx, vmptr);
return false;
page = nested_get_page(vcpu, vmcs12->msr_bitmap);
- if (!page) {
- WARN_ON(1);
+ if (!page)
return false;
- }
msr_bitmap_l1 = (unsigned long *)kmap(page);
memset(msr_bitmap_l0, 0xff, PAGE_SIZE);
{
struct vcpu_vmx *vmx = to_vmx(vcpu);
+ if (vcpu->arch.exception.pending ||
+ vcpu->arch.nmi_injected ||
+ vcpu->arch.interrupt.pending)
+ return -EBUSY;
+
if (nested_cpu_has_preemption_timer(get_vmcs12(vcpu)) &&
vmx->nested.preemption_timer_expired) {
if (vmx->nested.nested_run_pending)
}
if (vcpu->arch.nmi_pending && nested_exit_on_nmi(vcpu)) {
- if (vmx->nested.nested_run_pending ||
- vcpu->arch.interrupt.pending)
+ if (vmx->nested.nested_run_pending)
return -EBUSY;
nested_vmx_vmexit(vcpu, EXIT_REASON_EXCEPTION_NMI,
NMI_VECTOR | INTR_TYPE_NMI_INTR |
*/
static void vmx_leave_nested(struct kvm_vcpu *vcpu)
{
- if (is_guest_mode(vcpu))
+ if (is_guest_mode(vcpu)) {
+ to_vmx(vcpu)->nested.nested_run_pending = 0;
nested_vmx_vmexit(vcpu, -1, 0, 0);
+ }
free_nested(to_vmx(vcpu));
}
#include <linux/pvclock_gtod.h>
#include <linux/kvm_irqfd.h>
#include <linux/irqbypass.h>
+#include <linux/sched/stat.h>
+
#include <trace/events/kvm.h>
#include <asm/debugreg.h>
#include <linux/extable.h>
#include <linux/uaccess.h>
+#include <linux/sched/debug.h>
+
#include <asm/traps.h>
#include <asm/kdebug.h>
* Copyright (C) 2008-2009, Red Hat Inc., Ingo Molnar
*/
#include <linux/sched.h> /* test_thread_flag(), ... */
+#include <linux/sched/task_stack.h> /* task_stack_*(), ... */
#include <linux/kdebug.h> /* oops_begin/end, ... */
#include <linux/extable.h> /* search_exception_tables */
#include <linux/bootmem.h> /* max_low_pfn */
unsigned long end, int write, struct page **pages, int *nr)
{
struct dev_pagemap *pgmap = NULL;
- int nr_start = *nr;
- pte_t *ptep;
+ int nr_start = *nr, ret = 0;
+ pte_t *ptep, *ptem;
- ptep = pte_offset_map(&pmd, addr);
+ /*
+ * Keep the original mapped PTE value (ptem) around since we
+ * might increment ptep off the end of the page when finishing
+ * our loop iteration.
+ */
+ ptem = ptep = pte_offset_map(&pmd, addr);
do {
pte_t pte = gup_get_pte(ptep);
struct page *page;
/* Similar to the PMD case, NUMA hinting must take slow path */
- if (pte_protnone(pte)) {
- pte_unmap(ptep);
- return 0;
- }
+ if (pte_protnone(pte))
+ break;
+
+ if (!pte_allows_gup(pte_val(pte), write))
+ break;
if (pte_devmap(pte)) {
pgmap = get_dev_pagemap(pte_pfn(pte), pgmap);
if (unlikely(!pgmap)) {
undo_dev_pagemap(nr, nr_start, pages);
- pte_unmap(ptep);
- return 0;
+ break;
}
- } else if (!pte_allows_gup(pte_val(pte), write) ||
- pte_special(pte)) {
- pte_unmap(ptep);
- return 0;
- }
+ } else if (pte_special(pte))
+ break;
+
VM_BUG_ON(!pfn_valid(pte_pfn(pte)));
page = pte_page(pte);
get_page(page);
(*nr)++;
} while (ptep++, addr += PAGE_SIZE, addr != end);
- pte_unmap(ptep - 1);
+ if (addr == end)
+ ret = 1;
+ pte_unmap(ptem);
- return 1;
+ return ret;
}
static inline void get_head_page_multiple(struct page *page, int nr)
#include <linux/init.h>
#include <linux/fs.h>
#include <linux/mm.h>
+#include <linux/sched/mm.h>
#include <linux/hugetlb.h>
#include <linux/pagemap.h>
#include <linux/err.h>
#include <linux/kdebug.h>
#include <linux/mm.h>
#include <linux/sched.h>
+#include <linux/sched/task.h>
#include <linux/vmalloc.h>
#include <asm/tlbflush.h>
#include <linux/mm.h>
#include <linux/random.h>
#include <linux/limits.h>
-#include <linux/sched.h>
+#include <linux/sched/signal.h>
+#include <linux/sched/mm.h>
#include <asm/elf.h>
struct va_alignment __read_mostly va_align = {
*/
#include <linux/kernel.h>
#include <linux/slab.h>
+#include <linux/mm_types.h>
#include <linux/syscalls.h>
#include <linux/sched/sysctl.h>
pcibios_disable_irq(dev);
}
+#ifdef CONFIG_ACPI_HOTPLUG_IOAPIC
+void pcibios_release_device(struct pci_dev *dev)
+{
+ if (atomic_dec_return(&dev->enable_cnt) >= 0)
+ pcibios_disable_device(dev);
+
+}
+#endif
+
int pci_ext_cfg_avail(void)
{
if (raw_pci_ext_ops)
return 1;
for_each_pci_msi_entry(msidesc, dev) {
- __pci_read_msi_msg(msidesc, &msg);
- pirq = MSI_ADDR_EXT_DEST_ID(msg.address_hi) |
- ((msg.address_lo >> MSI_ADDR_DEST_ID_SHIFT) & 0xff);
- if (msg.data != XEN_PIRQ_MSI_DATA ||
- xen_irq_from_pirq(pirq) < 0) {
- pirq = xen_allocate_pirq_msi(dev, msidesc);
- if (pirq < 0) {
- irq = -ENODEV;
- goto error;
- }
- xen_msi_compose_msg(dev, pirq, &msg);
- __pci_write_msi_msg(msidesc, &msg);
- dev_dbg(&dev->dev, "xen: msi bound to pirq=%d\n", pirq);
- } else {
- dev_dbg(&dev->dev,
- "xen: msi already bound to pirq=%d\n", pirq);
+ pirq = xen_allocate_pirq_msi(dev, msidesc);
+ if (pirq < 0) {
+ irq = -ENODEV;
+ goto error;
}
+ xen_msi_compose_msg(dev, pirq, &msg);
+ __pci_write_msi_msg(msidesc, &msg);
+ dev_dbg(&dev->dev, "xen: msi bound to pirq=%d\n", pirq);
irq = xen_bind_pirq_msi_to_irq(dev, msidesc, pirq,
(type == PCI_CAP_ID_MSI) ? nvec : 1,
(type == PCI_CAP_ID_MSIX) ?
ops.write_payload_first(pnode, first);
ops.write_payload_last(pnode, last);
- ops.write_g_sw_ack(pnode, 0xffffUL);
/* in effect, all msg_type's are set to MSG_NOOP */
memset(pqp, 0, sizeof(struct bau_pq_entry) * DEST_Q_SIZE);
#include <linux/moduleparam.h>
#include <linux/nmi.h>
#include <linux/sched.h>
+#include <linux/sched/debug.h>
#include <linux/slab.h>
#include <linux/clocksource.h>
* Version 2. See the file COPYING for more details.
*/
+#include <linux/bug.h>
+#include <asm/purgatory.h>
+
#include "sha256.h"
#include "../boot/string.h"
-struct sha_region {
- unsigned long start;
- unsigned long len;
-};
-
-unsigned long backup_dest = 0;
-unsigned long backup_src = 0;
-unsigned long backup_sz = 0;
+unsigned long purgatory_backup_dest __section(.kexec-purgatory);
+unsigned long purgatory_backup_src __section(.kexec-purgatory);
+unsigned long purgatory_backup_sz __section(.kexec-purgatory);
-u8 sha256_digest[SHA256_DIGEST_SIZE] = { 0 };
+u8 purgatory_sha256_digest[SHA256_DIGEST_SIZE] __section(.kexec-purgatory);
-struct sha_region sha_regions[16] = {};
+struct kexec_sha_region purgatory_sha_regions[KEXEC_SEGMENT_MAX] __section(.kexec-purgatory);
/*
* On x86, second kernel requries first 640K of memory to boot. Copy
*/
static int copy_backup_region(void)
{
- if (backup_dest)
- memcpy((void *)backup_dest, (void *)backup_src, backup_sz);
-
+ if (purgatory_backup_dest) {
+ memcpy((void *)purgatory_backup_dest,
+ (void *)purgatory_backup_src, purgatory_backup_sz);
+ }
return 0;
}
-int verify_sha256_digest(void)
+static int verify_sha256_digest(void)
{
- struct sha_region *ptr, *end;
+ struct kexec_sha_region *ptr, *end;
u8 digest[SHA256_DIGEST_SIZE];
struct sha256_state sctx;
sha256_init(&sctx);
- end = &sha_regions[sizeof(sha_regions)/sizeof(sha_regions[0])];
- for (ptr = sha_regions; ptr < end; ptr++)
+ end = purgatory_sha_regions + ARRAY_SIZE(purgatory_sha_regions);
+
+ for (ptr = purgatory_sha_regions; ptr < end; ptr++)
sha256_update(&sctx, (uint8_t *)(ptr->start), ptr->len);
sha256_final(&sctx, digest);
- if (memcmp(digest, sha256_digest, sizeof(digest)))
+ if (memcmp(digest, purgatory_sha256_digest, sizeof(digest)))
return 1;
return 0;
* This source code is licensed under the GNU General Public License,
* Version 2. See the file COPYING for more details.
*/
+#include <asm/purgatory.h>
.text
.globl purgatory_start
#ifndef SHA256_H
#define SHA256_H
-
#include <linux/types.h>
#include <crypto/sha.h>
*/
#include <linux/sched.h>
+#include <linux/sched/mm.h>
#include <linux/uaccess.h>
#include <asm/prctl.h> /* XXX This should get the constants from libc */
#include <os.h>
#include <linux/kernel.h>
#include <linux/smp.h>
#include <linux/sched.h>
+#include <linux/sched/debug.h>
#include <linux/kallsyms.h>
#include <asm/ptrace.h>
#include <asm/sysrq.h>
#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/sched.h>
+#include <linux/sched/debug.h>
#include <linux/utsname.h>
#include <asm/current.h>
#include <asm/ptrace.h>
*
* Jeremy Fitzhardinge <jeremy@xensource.com>, XenSource Inc, 2007
*/
-#include <linux/sched.h>
+#include <linux/sched/mm.h>
#include <linux/highmem.h>
#include <linux/debugfs.h>
#include <linux/bug.h>
#include <linux/smp.h>
#include <linux/irq_work.h>
#include <linux/tick.h>
+#include <linux/nmi.h>
#include <asm/paravirt.h>
#include <asm/desc.h>
boot := arch/xtensa/boot
-all: zImage
-
-bzImage : zImage
-
-zImage: vmlinux
+all Image zImage uImage: vmlinux
$(Q)$(MAKE) $(build)=$(boot) $@
%.dtb:
$(Q)$(MAKE) $(build)=$(boot)/dts
define archhelp
+ @echo '* Image - Kernel ELF image with reset vector'
@echo '* zImage - Compressed kernel image (arch/xtensa/boot/images/zImage.*)'
+ @echo '* uImage - U-Boot wrapped image'
@echo ' dtbs - Build device tree blobs for enabled boards'
endef
# Subdirs for the boot loader(s)
-bootdir-$(CONFIG_XTENSA_PLATFORM_ISS) += boot-elf
-bootdir-$(CONFIG_XTENSA_PLATFORM_XT2000) += boot-redboot boot-elf boot-uboot
-bootdir-$(CONFIG_XTENSA_PLATFORM_XTFPGA) += boot-redboot boot-elf boot-uboot
+boot-$(CONFIG_XTENSA_PLATFORM_ISS) += Image
+boot-$(CONFIG_XTENSA_PLATFORM_XT2000) += Image zImage uImage
+boot-$(CONFIG_XTENSA_PLATFORM_XTFPGA) += Image zImage uImage
-zImage Image: $(bootdir-y)
+all: $(boot-y)
+Image: boot-elf
+zImage: boot-redboot
+uImage: $(obj)/uImage
-$(bootdir-y): $(addprefix $(obj)/,$(subdir-y)) \
- $(addprefix $(obj)/,$(host-progs))
+boot-elf boot-redboot: $(addprefix $(obj)/,$(subdir-y)) \
+ $(addprefix $(obj)/,$(host-progs))
$(Q)$(MAKE) $(build)=$(obj)/$@ $(MAKECMDGOALS)
OBJCOPYFLAGS = --strip-all -R .comment -R .note.gnu.build-id -O binary
boot-elf: vmlinux.bin
boot-redboot: vmlinux.bin.gz
-boot-uboot: vmlinux.bin.gz
+
+UIMAGE_LOADADDR = $(CONFIG_KERNEL_LOAD_ADDRESS)
+UIMAGE_COMPRESSION = gzip
+
+$(obj)/uImage: vmlinux.bin.gz FORCE
+ $(call if_changed,uimage)
+ $(Q)$(kecho) ' Kernel: $@ is ready'
-o $@ $(obj)/Image.o
$(Q)$(kecho) ' Kernel: $@ is ready'
-zImage: $(obj)/../Image.elf
+all Image: $(obj)/../Image.elf
$(Q)$(OBJCOPY) -S -O binary $< $@
$(Q)$(kecho) ' Kernel: $@ is ready'
-zImage: $(obj)/../zImage.redboot
+all zImage: $(obj)/../zImage.redboot
+++ /dev/null
-#
-# This file is subject to the terms and conditions of the GNU General Public
-# License. See the file "COPYING" in the main directory of this archive
-# for more details.
-#
-
-UIMAGE_LOADADDR = $(CONFIG_KERNEL_LOAD_ADDRESS)
-UIMAGE_COMPRESSION = gzip
-
-$(obj)/../uImage: vmlinux.bin.gz FORCE
- $(call if_changed,uimage)
- $(Q)$(kecho) ' Kernel: $@ is ready'
-
-zImage: $(obj)/../uImage
#include <linux/stringify.h>
#include <linux/sched.h>
+#include <linux/mm_types.h>
#include <asm/vectors.h>
#ifndef _XTENSA_PGTABLE_H
#define _XTENSA_PGTABLE_H
+#define __ARCH_USE_5LEVEL_HACK
#include <asm-generic/pgtable-nopmd.h>
#include <asm/page.h>
#include <asm/kmem_layout.h>
#endif /* CONFIG_MMU */
#define RESET_VECTOR1_VADDR (XCHAL_RESET_VECTOR1_VADDR)
+#ifdef CONFIG_VECTORS_OFFSET
#define VECBASE_VADDR (KERNELOFFSET - CONFIG_VECTORS_OFFSET)
+#else
+#define VECBASE_VADDR _vecbase
+#endif
#if defined(XCHAL_HAVE_VECBASE) && XCHAL_HAVE_VECBASE
#include <linux/errno.h>
#include <linux/sched.h>
+#include <linux/sched/debug.h>
+#include <linux/sched/task.h>
+#include <linux/sched/task_stack.h>
#include <linux/kernel.h>
#include <linux/mm.h>
#include <linux/smp.h>
#include <linux/perf_event.h>
#include <linux/ptrace.h>
#include <linux/sched.h>
+#include <linux/sched/task_stack.h>
#include <linux/security.h>
#include <linux/signal.h>
#include <linux/smp.h>
__tagtable(BP_TAG_INITRD, parse_tag_initrd);
+#endif /* CONFIG_BLK_DEV_INITRD */
+
#ifdef CONFIG_OF
static int __init parse_tag_fdt(const bp_tag_t *tag)
#endif /* CONFIG_OF */
-#endif /* CONFIG_BLK_DEV_INITRD */
-
static int __init parse_tag_cmdline(const bp_tag_t* tag)
{
strlcpy(command_line, (char *)(tag->data), COMMAND_LINE_SIZE);
mem_reserve(__pa(&_stext), __pa(&_end));
+#ifdef CONFIG_VECTORS_OFFSET
mem_reserve(__pa(&_WindowVectors_text_start),
__pa(&_WindowVectors_text_end));
__pa(&_Level6InterruptVector_text_end));
#endif
+#endif /* CONFIG_VECTORS_OFFSET */
+
#ifdef CONFIG_SMP
mem_reserve(__pa(&_SecondaryResetVector_text_start),
__pa(&_SecondaryResetVector_text_end));
#include <linux/ptrace.h>
#include <linux/personality.h>
#include <linux/tracehook.h>
+#include <linux/sched/task_stack.h>
#include <asm/ucontext.h>
#include <linux/uaccess.h>
#include <linux/irq.h>
#include <linux/kdebug.h>
#include <linux/module.h>
+#include <linux/sched/mm.h>
+#include <linux/sched/hotplug.h>
+#include <linux/sched/task_stack.h>
#include <linux/reboot.h>
#include <linux/seq_file.h>
#include <linux/smp.h>
#include <linux/file.h>
#include <linux/fs.h>
#include <linux/mman.h>
+#include <linux/sched/mm.h>
#include <linux/shm.h>
typedef void (*syscall_t)(void);
*/
#include <linux/kernel.h>
-#include <linux/sched.h>
+#include <linux/sched/signal.h>
+#include <linux/sched/debug.h>
+#include <linux/sched/task_stack.h>
#include <linux/init.h>
#include <linux/module.h>
#include <linux/stringify.h>
* garbage.)
*/
+#ifdef CONFIG_VECTORS_OFFSET
#define SECTION_VECTOR(sym, section, addr, max_prevsec_size, prevsec) \
section addr : AT((MIN(LOADADDR(prevsec) + max_prevsec_size, \
LOADADDR(prevsec) + SIZEOF(prevsec)) + 3) & ~ 3) \
*(section) \
sym ## _end = ABSOLUTE(.); \
}
+#else
+#define SECTION_VECTOR(section, addr) \
+ . = addr; \
+ *(section)
+#endif
/*
* Mapping of input sections to output sections when linking.
{
/* The HEAD_TEXT section must be the first section! */
HEAD_TEXT
+
+#ifndef CONFIG_VECTORS_OFFSET
+ . = ALIGN(PAGE_SIZE);
+ _vecbase = .;
+
+ SECTION_VECTOR (.WindowVectors.text, WINDOW_VECTORS_VADDR)
+#if XCHAL_EXCM_LEVEL >= 2
+ SECTION_VECTOR (.Level2InterruptVector.text, INTLEVEL2_VECTOR_VADDR)
+#endif
+#if XCHAL_EXCM_LEVEL >= 3
+ SECTION_VECTOR (.Level3InterruptVector.text, INTLEVEL3_VECTOR_VADDR)
+#endif
+#if XCHAL_EXCM_LEVEL >= 4
+ SECTION_VECTOR (.Level4InterruptVector.text, INTLEVEL4_VECTOR_VADDR)
+#endif
+#if XCHAL_EXCM_LEVEL >= 5
+ SECTION_VECTOR (.Level5InterruptVector.text, INTLEVEL5_VECTOR_VADDR)
+#endif
+#if XCHAL_EXCM_LEVEL >= 6
+ SECTION_VECTOR (.Level6InterruptVector.text, INTLEVEL6_VECTOR_VADDR)
+#endif
+ SECTION_VECTOR (.DebugInterruptVector.literal, DEBUG_VECTOR_VADDR - 4)
+ SECTION_VECTOR (.DebugInterruptVector.text, DEBUG_VECTOR_VADDR)
+ SECTION_VECTOR (.KernelExceptionVector.literal, KERNEL_VECTOR_VADDR - 4)
+ SECTION_VECTOR (.KernelExceptionVector.text, KERNEL_VECTOR_VADDR)
+ SECTION_VECTOR (.UserExceptionVector.literal, USER_VECTOR_VADDR - 4)
+ SECTION_VECTOR (.UserExceptionVector.text, USER_VECTOR_VADDR)
+ SECTION_VECTOR (.DoubleExceptionVector.literal, DOUBLEEXC_VECTOR_VADDR - 48)
+ SECTION_VECTOR (.DoubleExceptionVector.text, DOUBLEEXC_VECTOR_VADDR)
+#endif
+
TEXT_TEXT
VMLINUX_SYMBOL(__sched_text_start) = .;
*(.sched.literal .sched.text)
. = ALIGN(16);
__boot_reloc_table_start = ABSOLUTE(.);
+#ifdef CONFIG_VECTORS_OFFSET
RELOCATE_ENTRY(_WindowVectors_text,
.WindowVectors.text);
#if XCHAL_EXCM_LEVEL >= 2
.DoubleExceptionVector.text);
RELOCATE_ENTRY(_DebugInterruptVector_text,
.DebugInterruptVector.text);
+#endif
#if defined(CONFIG_SMP)
RELOCATE_ENTRY(_SecondaryResetVector_text,
.SecondaryResetVector.text);
. = ALIGN(4);
.dummy : { LONG(0) }
+#ifdef CONFIG_VECTORS_OFFSET
/* The vectors are relocated to the real position at startup time */
SECTION_VECTOR (_WindowVectors_text,
. = (LOADADDR( .DoubleExceptionVector.text ) + SIZEOF( .DoubleExceptionVector.text ) + 3) & ~ 3;
+#endif
#if defined(CONFIG_SMP)
SECTION_VECTOR (_SecondaryResetVector_text,
depends on BLOCK && PCI
default y
+config BLK_MQ_VIRTIO
+ bool
+ depends on BLOCK && VIRTIO
+ default y
+
source block/Kconfig.iosched
obj-$(CONFIG_BLK_CMDLINE_PARSER) += cmdline-parser.o
obj-$(CONFIG_BLK_DEV_INTEGRITY) += bio-integrity.o blk-integrity.o t10-pi.o
obj-$(CONFIG_BLK_MQ_PCI) += blk-mq-pci.o
+obj-$(CONFIG_BLK_MQ_VIRTIO) += blk-mq-virtio.o
obj-$(CONFIG_BLK_DEV_ZONED) += blk-zoned.o
obj-$(CONFIG_BLK_WBT) += blk-wbt.o
obj-$(CONFIG_BLK_DEBUG_FS) += blk-mq-debugfs.o
#include <linux/ioprio.h>
#include <linux/kdev_t.h>
#include <linux/module.h>
+#include <linux/sched/signal.h>
#include <linux/err.h>
#include <linux/blkdev.h>
#include <linux/backing-dev.h>
q->queue_lock = &q->__queue_lock;
spin_unlock_irq(lock);
- bdi_unregister(q->backing_dev_info);
- put_disk_devt(q->disk_devt);
-
/* @q is and will stay empty, shutdown and put */
blk_put_queue(q);
}
struct request_queue *q = bdev_get_queue(bio->bi_bdev);
if (likely(blk_queue_enter(q, false) == 0)) {
+ struct bio_list hold;
+ struct bio_list lower, same;
+
+ /* Create a fresh bio_list for all subordinate requests */
+ hold = bio_list_on_stack;
+ bio_list_init(&bio_list_on_stack);
ret = q->make_request_fn(q, bio);
blk_queue_exit(q);
- bio = bio_list_pop(current->bio_list);
+ /* sort new bios into those for a lower level
+ * and those for the same level
+ */
+ bio_list_init(&lower);
+ bio_list_init(&same);
+ while ((bio = bio_list_pop(&bio_list_on_stack)) != NULL)
+ if (q == bdev_get_queue(bio->bi_bdev))
+ bio_list_add(&same, bio);
+ else
+ bio_list_add(&lower, bio);
+ /* now assemble so we handle the lowest level first */
+ bio_list_merge(&bio_list_on_stack, &lower);
+ bio_list_merge(&bio_list_on_stack, &same);
+ bio_list_merge(&bio_list_on_stack, &hold);
} else {
- struct bio *bio_next = bio_list_pop(current->bio_list);
-
bio_io_error(bio);
- bio = bio_next;
}
+ bio = bio_list_pop(current->bio_list);
} while (bio);
current->bio_list = NULL; /* deactivate */
#include <linux/bio.h>
#include <linux/blkdev.h>
#include <linux/slab.h>
+#include <linux/sched/task.h>
#include "blk.h"
}
/*
- * Exit an icq. Called with both ioc and q locked for sq, only ioc locked for
- * mq.
+ * Exit an icq. Called with ioc locked for blk-mq, and with both ioc
+ * and queue locked for legacy.
*/
static void ioc_exit_icq(struct io_cq *icq)
{
icq->flags |= ICQ_EXITED;
}
-/* Release an icq. Called with both ioc and q locked. */
+/*
+ * Release an icq. Called with ioc locked for blk-mq, and with both ioc
+ * and queue locked for legacy.
+ */
static void ioc_destroy_icq(struct io_cq *icq)
{
struct io_context *ioc = icq->ioc;
struct elevator_type *et = q->elevator->type;
lockdep_assert_held(&ioc->lock);
- lockdep_assert_held(q->queue_lock);
radix_tree_delete(&ioc->icq_tree, icq->q->id);
hlist_del_init(&icq->ioc_node);
put_io_context_active(ioc);
}
+static void __ioc_clear_queue(struct list_head *icq_list)
+{
+ unsigned long flags;
+
+ while (!list_empty(icq_list)) {
+ struct io_cq *icq = list_entry(icq_list->next,
+ struct io_cq, q_node);
+ struct io_context *ioc = icq->ioc;
+
+ spin_lock_irqsave(&ioc->lock, flags);
+ ioc_destroy_icq(icq);
+ spin_unlock_irqrestore(&ioc->lock, flags);
+ }
+}
+
/**
* ioc_clear_queue - break any ioc association with the specified queue
* @q: request_queue being cleared
*
- * Walk @q->icq_list and exit all io_cq's. Must be called with @q locked.
+ * Walk @q->icq_list and exit all io_cq's.
*/
void ioc_clear_queue(struct request_queue *q)
{
- lockdep_assert_held(q->queue_lock);
+ LIST_HEAD(icq_list);
- while (!list_empty(&q->icq_list)) {
- struct io_cq *icq = list_entry(q->icq_list.next,
- struct io_cq, q_node);
- struct io_context *ioc = icq->ioc;
+ spin_lock_irq(q->queue_lock);
+ list_splice_init(&q->icq_list, &icq_list);
- spin_lock(&ioc->lock);
- ioc_destroy_icq(icq);
- spin_unlock(&ioc->lock);
+ if (q->mq_ops) {
+ spin_unlock_irq(q->queue_lock);
+ __ioc_clear_queue(&icq_list);
+ } else {
+ __ioc_clear_queue(&icq_list);
+ spin_unlock_irq(q->queue_lock);
}
}
* Functions related to mapping data to requests
*/
#include <linux/kernel.h>
+#include <linux/sched/task_stack.h>
#include <linux/module.h>
#include <linux/bio.h>
#include <linux/blkdev.h>
struct blk_mq_alloc_data *data)
{
struct elevator_queue *e = q->elevator;
- struct blk_mq_hw_ctx *hctx;
- struct blk_mq_ctx *ctx;
struct request *rq;
blk_queue_enter_live(q);
- ctx = blk_mq_get_ctx(q);
- hctx = blk_mq_map_queue(q, ctx->cpu);
-
- blk_mq_set_alloc_data(data, q, data->flags, ctx, hctx);
+ data->q = q;
+ if (likely(!data->ctx))
+ data->ctx = blk_mq_get_ctx(q);
+ if (likely(!data->hctx))
+ data->hctx = blk_mq_map_queue(q, data->ctx->cpu);
if (e) {
data->flags |= BLK_MQ_REQ_INTERNAL;
rq = __blk_mq_alloc_request(data, op);
} else {
rq = __blk_mq_alloc_request(data, op);
- if (rq)
- data->hctx->tags->rqs[rq->tag] = rq;
}
if (rq) {
*/
ret = 0;
queue_for_each_hw_ctx(q, hctx, i) {
- hctx->sched_tags = blk_mq_alloc_rq_map(set, i, q->nr_requests, 0);
+ hctx->sched_tags = blk_mq_alloc_rq_map(set, i,
+ q->nr_requests, set->reserved_tags);
if (!hctx->sched_tags) {
ret = -ENOMEM;
break;
{
}
+static void blk_mq_hw_sysfs_release(struct kobject *kobj)
+{
+ struct blk_mq_hw_ctx *hctx = container_of(kobj, struct blk_mq_hw_ctx,
+ kobj);
+ free_cpumask_var(hctx->cpumask);
+ kfree(hctx->ctxs);
+ kfree(hctx);
+}
+
struct blk_mq_ctx_sysfs_entry {
struct attribute attr;
ssize_t (*show)(struct blk_mq_ctx *, char *);
static struct kobj_type blk_mq_hw_ktype = {
.sysfs_ops = &blk_mq_hw_sysfs_ops,
.default_attrs = default_hw_ctx_attrs,
- .release = blk_mq_sysfs_release,
+ .release = blk_mq_hw_sysfs_release,
};
static void blk_mq_unregister_hctx(struct blk_mq_hw_ctx *hctx)
static void __blk_mq_unregister_dev(struct device *dev, struct request_queue *q)
{
struct blk_mq_hw_ctx *hctx;
- struct blk_mq_ctx *ctx;
- int i, j;
+ int i;
- queue_for_each_hw_ctx(q, hctx, i) {
+ queue_for_each_hw_ctx(q, hctx, i)
blk_mq_unregister_hctx(hctx);
- hctx_for_each_ctx(hctx, ctx, j)
- kobject_put(&ctx->kobj);
-
- kobject_put(&hctx->kobj);
- }
-
blk_mq_debugfs_unregister_hctxs(q);
kobject_uevent(&q->mq_kobj, KOBJ_REMOVE);
kobject_del(&q->mq_kobj);
- kobject_put(&q->mq_kobj);
-
kobject_put(&dev->kobj);
q->mq_sysfs_init_done = false;
kobject_init(&hctx->kobj, &blk_mq_hw_ktype);
}
-static void blk_mq_sysfs_init(struct request_queue *q)
+void blk_mq_sysfs_deinit(struct request_queue *q)
+{
+ struct blk_mq_ctx *ctx;
+ int cpu;
+
+ for_each_possible_cpu(cpu) {
+ ctx = per_cpu_ptr(q->queue_ctx, cpu);
+ kobject_put(&ctx->kobj);
+ }
+ kobject_put(&q->mq_kobj);
+}
+
+void blk_mq_sysfs_init(struct request_queue *q)
{
struct blk_mq_ctx *ctx;
int cpu;
blk_mq_disable_hotplug();
- blk_mq_sysfs_init(q);
-
ret = kobject_add(&q->mq_kobj, kobject_get(&dev->kobj), "%s", "mq");
if (ret < 0)
goto out;
void blk_mq_put_tag(struct blk_mq_hw_ctx *hctx, struct blk_mq_tags *tags,
struct blk_mq_ctx *ctx, unsigned int tag)
{
- if (tag >= tags->nr_reserved_tags) {
+ if (!blk_mq_tag_is_reserved(tags, tag)) {
const int real_tag = tag - tags->nr_reserved_tags;
BUG_ON(real_tag >= tags->nr_tags);
hctx->tags->rqs[tag] = rq;
}
+static inline bool blk_mq_tag_is_reserved(struct blk_mq_tags *tags,
+ unsigned int tag)
+{
+ return tag < tags->nr_reserved_tags;
+}
+
#endif
--- /dev/null
+/*
+ * Copyright (c) 2016 Christoph Hellwig.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ */
+#include <linux/device.h>
+#include <linux/blk-mq.h>
+#include <linux/blk-mq-virtio.h>
+#include <linux/virtio_config.h>
+#include <linux/module.h>
+#include "blk-mq.h"
+
+/**
+ * blk_mq_virtio_map_queues - provide a default queue mapping for virtio device
+ * @set: tagset to provide the mapping for
+ * @vdev: virtio device associated with @set.
+ * @first_vec: first interrupt vectors to use for queues (usually 0)
+ *
+ * This function assumes the virtio device @vdev has at least as many available
+ * interrupt vetors as @set has queues. It will then queuery the vector
+ * corresponding to each queue for it's affinity mask and built queue mapping
+ * that maps a queue to the CPUs that have irq affinity for the corresponding
+ * vector.
+ */
+int blk_mq_virtio_map_queues(struct blk_mq_tag_set *set,
+ struct virtio_device *vdev, int first_vec)
+{
+ const struct cpumask *mask;
+ unsigned int queue, cpu;
+
+ if (!vdev->config->get_vq_affinity)
+ goto fallback;
+
+ for (queue = 0; queue < set->nr_hw_queues; queue++) {
+ mask = vdev->config->get_vq_affinity(vdev, first_vec + queue);
+ if (!mask)
+ goto fallback;
+
+ for_each_cpu(cpu, mask)
+ set->mq_map[cpu] = queue;
+ }
+
+ return 0;
+fallback:
+ return blk_mq_map_queues(set);
+}
+EXPORT_SYMBOL_GPL(blk_mq_virtio_map_queues);
#include <linux/cpu.h>
#include <linux/cache.h>
#include <linux/sched/sysctl.h>
+#include <linux/sched/topology.h>
+#include <linux/sched/signal.h>
#include <linux/delay.h>
#include <linux/crash_dump.h>
#include <linux/prefetch.h>
}
EXPORT_SYMBOL_GPL(blk_mq_freeze_queue_start);
-static void blk_mq_freeze_queue_wait(struct request_queue *q)
+void blk_mq_freeze_queue_wait(struct request_queue *q)
{
wait_event(q->mq_freeze_wq, percpu_ref_is_zero(&q->q_usage_counter));
}
+EXPORT_SYMBOL_GPL(blk_mq_freeze_queue_wait);
+
+int blk_mq_freeze_queue_wait_timeout(struct request_queue *q,
+ unsigned long timeout)
+{
+ return wait_event_timeout(q->mq_freeze_wq,
+ percpu_ref_is_zero(&q->q_usage_counter),
+ timeout);
+}
+EXPORT_SYMBOL_GPL(blk_mq_freeze_queue_wait_timeout);
/*
* Guarantee no request is in use, so we can change any data structure of
}
rq->tag = tag;
rq->internal_tag = -1;
+ data->hctx->tags->rqs[rq->tag] = rq;
}
blk_mq_rq_ctx_init(data->q, data->ctx, rq, op);
struct request *blk_mq_alloc_request_hctx(struct request_queue *q, int rw,
unsigned int flags, unsigned int hctx_idx)
{
- struct blk_mq_hw_ctx *hctx;
- struct blk_mq_ctx *ctx;
+ struct blk_mq_alloc_data alloc_data = { .flags = flags };
struct request *rq;
- struct blk_mq_alloc_data alloc_data;
+ unsigned int cpu;
int ret;
/*
* Check if the hardware context is actually mapped to anything.
* If not tell the caller that it should skip this queue.
*/
- hctx = q->queue_hw_ctx[hctx_idx];
- if (!blk_mq_hw_queue_mapped(hctx)) {
- ret = -EXDEV;
- goto out_queue_exit;
- }
- ctx = __blk_mq_get_ctx(q, cpumask_first(hctx->cpumask));
-
- blk_mq_set_alloc_data(&alloc_data, q, flags, ctx, hctx);
- rq = __blk_mq_alloc_request(&alloc_data, rw);
- if (!rq) {
- ret = -EWOULDBLOCK;
- goto out_queue_exit;
+ alloc_data.hctx = q->queue_hw_ctx[hctx_idx];
+ if (!blk_mq_hw_queue_mapped(alloc_data.hctx)) {
+ blk_queue_exit(q);
+ return ERR_PTR(-EXDEV);
}
+ cpu = cpumask_first(alloc_data.hctx->cpumask);
+ alloc_data.ctx = __blk_mq_get_ctx(q, cpu);
- return rq;
+ rq = blk_mq_sched_get_request(q, NULL, rw, &alloc_data);
-out_queue_exit:
+ blk_mq_put_ctx(alloc_data.ctx);
blk_queue_exit(q);
- return ERR_PTR(ret);
+
+ if (!rq)
+ return ERR_PTR(-EWOULDBLOCK);
+
+ return rq;
}
EXPORT_SYMBOL_GPL(blk_mq_alloc_request_hctx);
return true;
}
+ if (blk_mq_tag_is_reserved(data.hctx->sched_tags, rq->internal_tag))
+ data.flags |= BLK_MQ_REQ_RESERVED;
+
rq->tag = blk_mq_get_tag(&data);
if (rq->tag >= 0) {
if (blk_mq_tag_busy(data.hctx)) {
return false;
}
-static void blk_mq_put_driver_tag(struct blk_mq_hw_ctx *hctx,
- struct request *rq)
+static void __blk_mq_put_driver_tag(struct blk_mq_hw_ctx *hctx,
+ struct request *rq)
{
- if (rq->tag == -1 || rq->internal_tag == -1)
- return;
-
blk_mq_put_tag(hctx, hctx->tags, rq->mq_ctx, rq->tag);
rq->tag = -1;
}
}
+static void blk_mq_put_driver_tag_hctx(struct blk_mq_hw_ctx *hctx,
+ struct request *rq)
+{
+ if (rq->tag == -1 || rq->internal_tag == -1)
+ return;
+
+ __blk_mq_put_driver_tag(hctx, rq);
+}
+
+static void blk_mq_put_driver_tag(struct request *rq)
+{
+ struct blk_mq_hw_ctx *hctx;
+
+ if (rq->tag == -1 || rq->internal_tag == -1)
+ return;
+
+ hctx = blk_mq_map_queue(rq->q, rq->mq_ctx->cpu);
+ __blk_mq_put_driver_tag(hctx, rq);
+}
+
/*
* If we fail getting a driver tag because all the driver tags are already
* assigned and on the dispatch list, BUT the first entry does not have a
bd.rq = rq;
bd.list = dptr;
- bd.last = list_empty(list);
+
+ /*
+ * Flag last if we have no more requests, or if we have more
+ * but can't assign a driver tag to it.
+ */
+ if (list_empty(list))
+ bd.last = true;
+ else {
+ struct request *nxt;
+
+ nxt = list_first_entry(list, struct request, queuelist);
+ bd.last = !blk_mq_get_driver_tag(nxt, NULL, false);
+ }
ret = q->mq_ops->queue_rq(hctx, &bd);
switch (ret) {
queued++;
break;
case BLK_MQ_RQ_QUEUE_BUSY:
- blk_mq_put_driver_tag(hctx, rq);
+ blk_mq_put_driver_tag_hctx(hctx, rq);
list_add(&rq->queuelist, list);
__blk_mq_requeue_request(rq);
break;
* that is where we will continue on next queue run.
*/
if (!list_empty(list)) {
+ /*
+ * If we got a driver tag for the next request already,
+ * free it again.
+ */
+ rq = list_first_entry(list, struct request, queuelist);
+ blk_mq_put_driver_tag(rq);
+
spin_lock(&hctx->lock);
list_splice_init(list, &hctx->dispatch);
spin_unlock(&hctx->lock);
unsigned int reserved_tags)
{
struct blk_mq_tags *tags;
+ int node;
+
+ node = blk_mq_hw_queue_to_node(set->mq_map, hctx_idx);
+ if (node == NUMA_NO_NODE)
+ node = set->numa_node;
- tags = blk_mq_init_tags(nr_tags, reserved_tags,
- set->numa_node,
+ tags = blk_mq_init_tags(nr_tags, reserved_tags, node,
BLK_MQ_FLAG_TO_ALLOC_POLICY(set->flags));
if (!tags)
return NULL;
tags->rqs = kzalloc_node(nr_tags * sizeof(struct request *),
GFP_NOIO | __GFP_NOWARN | __GFP_NORETRY,
- set->numa_node);
+ node);
if (!tags->rqs) {
blk_mq_free_tags(tags);
return NULL;
tags->static_rqs = kzalloc_node(nr_tags * sizeof(struct request *),
GFP_NOIO | __GFP_NOWARN | __GFP_NORETRY,
- set->numa_node);
+ node);
if (!tags->static_rqs) {
kfree(tags->rqs);
blk_mq_free_tags(tags);
{
unsigned int i, j, entries_per_page, max_order = 4;
size_t rq_size, left;
+ int node;
+
+ node = blk_mq_hw_queue_to_node(set->mq_map, hctx_idx);
+ if (node == NUMA_NO_NODE)
+ node = set->numa_node;
INIT_LIST_HEAD(&tags->page_list);
this_order--;
do {
- page = alloc_pages_node(set->numa_node,
+ page = alloc_pages_node(node,
GFP_NOIO | __GFP_NOWARN | __GFP_NORETRY | __GFP_ZERO,
this_order);
if (page)
if (set->ops->init_request) {
if (set->ops->init_request(set->driver_data,
rq, hctx_idx, i,
- set->numa_node)) {
+ node)) {
tags->static_rqs[i] = NULL;
goto fail;
}
}
}
-static void blk_mq_free_hw_queues(struct request_queue *q,
- struct blk_mq_tag_set *set)
-{
- struct blk_mq_hw_ctx *hctx;
- unsigned int i;
-
- queue_for_each_hw_ctx(q, hctx, i)
- free_cpumask_var(hctx->cpumask);
-}
-
static int blk_mq_init_hctx(struct request_queue *q,
struct blk_mq_tag_set *set,
struct blk_mq_hw_ctx *hctx, unsigned hctx_idx)
struct blk_mq_ctx *__ctx = per_cpu_ptr(q->queue_ctx, i);
struct blk_mq_hw_ctx *hctx;
- memset(__ctx, 0, sizeof(*__ctx));
__ctx->cpu = i;
spin_lock_init(&__ctx->lock);
INIT_LIST_HEAD(&__ctx->rq_list);
queue_for_each_hw_ctx(q, hctx, i) {
if (!hctx)
continue;
- kfree(hctx->ctxs);
- kfree(hctx);
+ kobject_put(&hctx->kobj);
}
q->mq_map = NULL;
kfree(q->queue_hw_ctx);
- /* ctx kobj stays in queue_ctx */
+ /*
+ * release .mq_kobj and sw queue's kobject now because
+ * both share lifetime with request queue.
+ */
+ blk_mq_sysfs_deinit(q);
+
free_percpu(q->queue_ctx);
}
if (hctx->tags)
blk_mq_free_map_and_requests(set, j);
blk_mq_exit_hctx(q, set, hctx, j);
- free_cpumask_var(hctx->cpumask);
kobject_put(&hctx->kobj);
- kfree(hctx->ctxs);
- kfree(hctx);
hctxs[j] = NULL;
}
if (!q->queue_ctx)
goto err_exit;
+ /* init q->mq_kobj and sw queues' kobjects */
+ blk_mq_sysfs_init(q);
+
q->queue_hw_ctx = kzalloc_node(nr_cpu_ids * sizeof(*(q->queue_hw_ctx)),
GFP_KERNEL, set->numa_node);
if (!q->queue_hw_ctx)
blk_mq_del_queue_tag_set(q);
blk_mq_exit_hw_queues(q, set, set->nr_hw_queues);
- blk_mq_free_hw_queues(q, set);
}
/* Basically redo blk_mq_init_queue with queue frozen */
/*
* sysfs helpers
*/
+extern void blk_mq_sysfs_init(struct request_queue *q);
+extern void blk_mq_sysfs_deinit(struct request_queue *q);
extern int blk_mq_sysfs_register(struct request_queue *q);
extern void blk_mq_sysfs_unregister(struct request_queue *q);
extern void blk_mq_hctx_kobj_init(struct blk_mq_hw_ctx *hctx);
struct blk_mq_hw_ctx *hctx;
};
-static inline void blk_mq_set_alloc_data(struct blk_mq_alloc_data *data,
- struct request_queue *q, unsigned int flags,
- struct blk_mq_ctx *ctx, struct blk_mq_hw_ctx *hctx)
-{
- data->q = q;
- data->flags = flags;
- data->ctx = ctx;
- data->hctx = hctx;
-}
-
static inline struct blk_mq_tags *blk_mq_tags_from_data(struct blk_mq_alloc_data *data)
{
if (data->flags & BLK_MQ_REQ_INTERNAL)
#include <linux/interrupt.h>
#include <linux/cpu.h>
#include <linux/sched.h>
+#include <linux/sched/topology.h>
#include "blk.h"
blkcg_exit_queue(q);
if (q->elevator) {
- spin_lock_irq(q->queue_lock);
ioc_clear_queue(q);
- spin_unlock_irq(q->queue_lock);
elevator_exit(q->elevator);
}
*/
#include <linux/module.h>
#include <linux/slab.h>
+#include <linux/sched/clock.h>
#include <linux/blkdev.h>
#include <linux/elevator.h>
#include <linux/ktime.h>
if (old_registered)
elv_unregister_queue(q);
- spin_lock_irq(q->queue_lock);
ioc_clear_queue(q);
- spin_unlock_irq(q->queue_lock);
}
/* allocate, init and register new elevator */
disk_part_iter_exit(&piter);
}
-void put_disk_devt(struct disk_devt *disk_devt)
-{
- if (disk_devt && atomic_dec_and_test(&disk_devt->count))
- disk_devt->release(disk_devt);
-}
-EXPORT_SYMBOL(put_disk_devt);
-
-void get_disk_devt(struct disk_devt *disk_devt)
-{
- if (disk_devt)
- atomic_inc(&disk_devt->count);
-}
-EXPORT_SYMBOL(get_disk_devt);
-
/**
* device_add_disk - add partitioning information to kernel list
* @parent: parent device for the disk
disk_alloc_events(disk);
- /*
- * Take a reference on the devt and assign it to queue since it
- * must not be reallocated while the bdi is registered
- */
- disk->queue->disk_devt = disk->disk_devt;
- get_disk_devt(disk->disk_devt);
-
/* Register BDI before referencing it from bdev */
bdi = disk->queue->backing_dev_info;
bdi_register_owner(bdi, disk_to_dev(disk));
disk->flags &= ~GENHD_FL_UP;
sysfs_remove_link(&disk_to_dev(disk)->kobj, "bdi");
- blk_unregister_queue(disk);
+ if (disk->queue) {
+ /*
+ * Unregister bdi before releasing device numbers (as they can
+ * get reused and we'd get clashes in sysfs).
+ */
+ bdi_unregister(disk->queue->backing_dev_info);
+ blk_unregister_queue(disk);
+ } else {
+ WARN_ON(1);
+ }
blk_unregister_region(disk_devt(disk), disk->minors);
part_stat_set_all(&disk->part0, 0);
#include <linux/kernel.h>
#include <linux/export.h>
#include <linux/ioprio.h>
+#include <linux/cred.h>
#include <linux/blkdev.h>
#include <linux/capability.h>
+#include <linux/sched/user.h>
+#include <linux/sched/task.h>
#include <linux/syscalls.h>
#include <linux/security.h>
#include <linux/pid_namespace.h>
static int gen_key(struct opal_dev *dev, void *data)
{
- const u8 *method;
u8 uid[OPAL_UID_LENGTH];
int err = 0;
set_comid(dev, dev->comid);
memcpy(uid, dev->prev_data, min(sizeof(uid), dev->prev_d_len));
- method = opalmethod[OPAL_GENKEY];
kfree(dev->prev_data);
dev->prev_data = NULL;
static int lock_unlock_locking_range(struct opal_dev *dev, void *data)
{
u8 lr_buffer[OPAL_UID_LENGTH];
- const u8 *method;
struct opal_lock_unlock *lkul = data;
u8 read_locked = 1, write_locked = 1;
int err = 0;
clear_opal_cmd(dev);
set_comid(dev, dev->comid);
- method = opalmethod[OPAL_SET];
if (build_locking_range(lr_buffer, sizeof(lr_buffer),
lkul->session.opal_key.lr) < 0)
return -ERANGE;
{
u8 lr_buffer[OPAL_UID_LENGTH];
u8 read_locked = 1, write_locked = 1;
- const u8 *method;
struct opal_lock_unlock *lkul = data;
int ret;
clear_opal_cmd(dev);
set_comid(dev, dev->comid);
- method = opalmethod[OPAL_SET];
if (build_locking_range(lr_buffer, sizeof(lr_buffer),
lkul->session.opal_key.lr) < 0)
return -ERANGE;
pr_err("Locking state was not RO or RW\n");
return -EINVAL;
}
- if (lk_unlk->session.who < OPAL_USER1 &&
+ if (lk_unlk->session.who < OPAL_USER1 ||
lk_unlk->session.who > OPAL_USER9) {
pr_err("Authority was not within the range of users: %d\n",
lk_unlk->session.who);
int ret;
/* We can't activate Admin1 it's active as manufactured */
- if (opal_session->who < OPAL_USER1 &&
+ if (opal_session->who < OPAL_USER1 ||
opal_session->who > OPAL_USER9) {
pr_err("Who was not a valid user: %d\n", opal_session->who);
return -EINVAL;
#include <linux/module.h>
#include <linux/notifier.h>
#include <linux/rtnetlink.h>
-#include <linux/sched.h>
+#include <linux/sched/signal.h>
#include <linux/slab.h>
#include <linux/string.h>
#include <linux/init.h>
#include <linux/list.h>
#include <linux/kernel.h>
+#include <linux/sched/signal.h>
#include <linux/mm.h>
#include <linux/module.h>
#include <linux/net.h>
#include <linux/init.h>
#include <linux/list.h>
#include <linux/kernel.h>
+#include <linux/sched/signal.h>
#include <linux/mm.h>
#include <linux/module.h>
#include <linux/net.h>
#include <linux/kmod.h>
#include <linux/module.h>
#include <linux/param.h>
-#include <linux/sched.h>
+#include <linux/sched/signal.h>
#include <linux/slab.h>
#include <linux/string.h>
#include "internal.h"
struct crypto_ccm_req_priv_ctx {
u8 odata[16];
+ u8 idata[16];
u8 auth_tag[16];
u32 flags;
struct scatterlist src[3];
AHASH_REQUEST_ON_STACK(ahreq, ctx->mac);
unsigned int assoclen = req->assoclen;
struct scatterlist sg[3];
- u8 odata[16];
- u8 idata[16];
+ u8 *odata = pctx->odata;
+ u8 *idata = pctx->idata;
int ilen, err;
/* format control data for input */
#include <linux/delay.h>
#include <crypto/engine.h>
#include <crypto/internal/hash.h>
+#include <uapi/linux/sched/types.h>
#include "internal.h"
#define CRYPTO_ENGINE_MAX_QLEN 10
#include <linux/module.h>
#include <linux/scatterlist.h>
#include <linux/sched.h>
+#include <linux/sched/stat.h>
#include <linux/slab.h>
#include <linux/hardirq.h>
"\x09\x75\x9a\x9b\x3c\x9b\x27\x39",
.klen = 32,
.iv = "\x03\xf9\xd9\x4e\x63\xb5\x3d\x9d"
- "\x43\xf6\x1e\x50",
+ "\x43\xf6\x1e\x50\0\0\0\0",
.assoc = "\x57\xf5\x6b\x8b\x57\x5c\x3d\x3b"
"\x13\x02\x01\x0c\x83\x4c\x96\x35"
"\x8e\xd6\x39\xcf\x7d\x14\x9b\x94"
struct xts_instance_ctx *ctx;
struct skcipher_alg *alg;
const char *cipher_name;
+ u32 mask;
int err;
algt = crypto_get_attr_type(tb);
ctx = skcipher_instance_ctx(inst);
crypto_set_skcipher_spawn(&ctx->spawn, skcipher_crypto_instance(inst));
- err = crypto_grab_skcipher(&ctx->spawn, cipher_name, 0,
- crypto_requires_sync(algt->type,
- algt->mask));
+
+ mask = crypto_requires_off(algt->type, algt->mask,
+ CRYPTO_ALG_NEED_FALLBACK |
+ CRYPTO_ALG_ASYNC);
+
+ err = crypto_grab_skcipher(&ctx->spawn, cipher_name, 0, mask);
if (err == -ENOENT) {
err = -ENAMETOOLONG;
if (snprintf(ctx->name, CRYPTO_MAX_ALG_NAME, "ecb(%s)",
cipher_name) >= CRYPTO_MAX_ALG_NAME)
goto err_free_inst;
- err = crypto_grab_skcipher(&ctx->spawn, ctx->name, 0,
- crypto_requires_sync(algt->type,
- algt->mask));
+ err = crypto_grab_skcipher(&ctx->spawn, ctx->name, 0, mask);
}
if (err)
#include <linux/init.h>
#include <linux/types.h>
#include <linux/kthread.h>
+#include <uapi/linux/sched/types.h>
#include <linux/freezer.h>
#include <linux/cpu.h>
#include <linux/tick.h>
static int __init bert_init(void)
{
+ struct apei_resources bert_resources;
struct acpi_bert_region *boot_error_region;
struct acpi_table_bert *bert_tab;
unsigned int region_len;
}
region_len = bert_tab->region_length;
- if (!request_mem_region(bert_tab->address, region_len, "APEI BERT")) {
- pr_err("Can't request iomem region <%016llx-%016llx>.\n",
- (unsigned long long)bert_tab->address,
- (unsigned long long)bert_tab->address + region_len - 1);
- return -EIO;
- }
-
+ apei_resources_init(&bert_resources);
+ rc = apei_resources_add(&bert_resources, bert_tab->address,
+ region_len, true);
+ if (rc)
+ return rc;
+ rc = apei_resources_request(&bert_resources, "APEI BERT");
+ if (rc)
+ goto out_fini;
boot_error_region = ioremap_cache(bert_tab->address, region_len);
if (boot_error_region) {
bert_print_all(boot_error_region, region_len);
rc = -ENOMEM;
}
- release_mem_region(bert_tab->address, region_len);
+ apei_resources_release(&bert_resources);
+out_fini:
+ apei_resources_fini(&bert_resources);
return rc;
}
#include <linux/pci.h>
#include <linux/aer.h>
#include <linux/nmi.h>
+#include <linux/sched/clock.h>
#include <acpi/ghes.h>
#include <acpi/apei.h>
void acpi_container_init(void);
void acpi_memory_hotplug_init(void);
#ifdef CONFIG_ACPI_HOTPLUG_IOAPIC
+void pci_ioapic_remove(struct acpi_pci_root *root);
int acpi_ioapic_remove(struct acpi_pci_root *root);
#else
+static inline void pci_ioapic_remove(struct acpi_pci_root *root) { return; }
static inline int acpi_ioapic_remove(struct acpi_pci_root *root) { return 0; }
#endif
#ifdef CONFIG_ACPI_DOCK
return ACPI_SUCCESS(status) && ACPI_SUCCESS(retval) ? 0 : -ENODEV;
}
-int acpi_ioapic_remove(struct acpi_pci_root *root)
+void pci_ioapic_remove(struct acpi_pci_root *root)
{
- int retval = 0;
struct acpi_pci_ioapic *ioapic, *tmp;
mutex_lock(&ioapic_list_lock);
list_for_each_entry_safe(ioapic, tmp, &ioapic_list, list) {
if (root->device->handle != ioapic->root_handle)
continue;
-
- if (acpi_unregister_ioapic(ioapic->handle, ioapic->gsi_base))
- retval = -EBUSY;
-
if (ioapic->pdev) {
pci_release_region(ioapic->pdev, 0);
pci_disable_device(ioapic->pdev);
pci_dev_put(ioapic->pdev);
}
+ }
+ mutex_unlock(&ioapic_list_lock);
+}
+
+int acpi_ioapic_remove(struct acpi_pci_root *root)
+{
+ int retval = 0;
+ struct acpi_pci_ioapic *ioapic, *tmp;
+
+ mutex_lock(&ioapic_list_lock);
+ list_for_each_entry_safe(ioapic, tmp, &ioapic_list, list) {
+ if (root->device->handle != ioapic->root_handle)
+ continue;
+ if (acpi_unregister_ioapic(ioapic->handle, ioapic->gsi_base))
+ retval = -EBUSY;
if (ioapic->res.flags && ioapic->res.parent)
release_resource(&ioapic->res);
list_del(&ioapic->list);
+ num_mappings * sizeof(struct nfit_set_info_map);
}
-static int cmp_map(const void *m0, const void *m1)
+static int cmp_map_compat(const void *m0, const void *m1)
{
const struct nfit_set_info_map *map0 = m0;
const struct nfit_set_info_map *map1 = m1;
sizeof(u64));
}
+static int cmp_map(const void *m0, const void *m1)
+{
+ const struct nfit_set_info_map *map0 = m0;
+ const struct nfit_set_info_map *map1 = m1;
+
+ return map0->region_offset - map1->region_offset;
+}
+
/* Retrieve the nth entry referencing this spa */
static struct acpi_nfit_memory_map *memdev_from_spa(
struct acpi_nfit_desc *acpi_desc, u16 range_index, int n)
sort(&info->mapping[0], nr, sizeof(struct nfit_set_info_map),
cmp_map, NULL);
nd_set->cookie = nd_fletcher64(info, sizeof_nfit_set_info(nr), 0);
+
+ /* support namespaces created with the wrong sort order */
+ sort(&info->mapping[0], nr, sizeof(struct nfit_set_info_map),
+ cmp_map_compat, NULL);
+ nd_set->altcookie = nd_fletcher64(info, sizeof_nfit_set_info(nr), 0);
+
ndr_desc->nd_set = nd_set;
devm_kfree(dev, info);
pci_stop_root_bus(root->bus);
- WARN_ON(acpi_ioapic_remove(root));
-
+ pci_ioapic_remove(root);
device_set_run_wake(root->bus->bridge, false);
pci_acpi_remove_bus_pm_notifier(device);
pci_remove_root_bus(root->bus);
+ WARN_ON(acpi_ioapic_remove(root));
dmar_device_remove(device->handle);
#include <linux/poll.h>
#include <linux/debugfs.h>
#include <linux/rbtree.h>
-#include <linux/sched.h>
+#include <linux/sched/signal.h>
+#include <linux/sched/mm.h>
#include <linux/seq_file.h>
#include <linux/uaccess.h>
#include <linux/vmalloc.h>
u8 args[4], *argbuf = NULL, *sensebuf = NULL;
int argsize = 0;
enum dma_data_direction data_dir;
+ struct scsi_sense_hdr sshdr;
int cmd_result;
if (arg == NULL)
/* Good values for timeout and retries? Values below
from scsi_ioctl_send_command() for default case... */
cmd_result = scsi_execute(scsidev, scsi_cmd, data_dir, argbuf, argsize,
- sensebuf, (10*HZ), 5, 0, NULL);
+ sensebuf, &sshdr, (10*HZ), 5, 0, 0, NULL);
if (driver_byte(cmd_result) == DRIVER_SENSE) {/* sense data available */
u8 *desc = sensebuf + 8;
/* If we set cc then ATA pass-through will cause a
* check condition even if no error. Filter that. */
if (cmd_result & SAM_STAT_CHECK_CONDITION) {
- struct scsi_sense_hdr sshdr;
- scsi_normalize_sense(sensebuf, SCSI_SENSE_BUFFERSIZE,
- &sshdr);
if (sshdr.sense_key == RECOVERED_ERROR &&
sshdr.asc == 0 && sshdr.ascq == 0x1d)
cmd_result &= ~SAM_STAT_CHECK_CONDITION;
int rc = 0;
u8 scsi_cmd[MAX_COMMAND_SIZE];
u8 args[7], *sensebuf = NULL;
+ struct scsi_sense_hdr sshdr;
int cmd_result;
if (arg == NULL)
/* Good values for timeout and retries? Values below
from scsi_ioctl_send_command() for default case... */
cmd_result = scsi_execute(scsidev, scsi_cmd, DMA_NONE, NULL, 0,
- sensebuf, (10*HZ), 5, 0, NULL);
+ sensebuf, &sshdr, (10*HZ), 5, 0, 0, NULL);
if (driver_byte(cmd_result) == DRIVER_SENSE) {/* sense data available */
u8 *desc = sensebuf + 8;
/* If we set cc then ATA pass-through will cause a
* check condition even if no error. Filter that. */
if (cmd_result & SAM_STAT_CHECK_CONDITION) {
- struct scsi_sense_hdr sshdr;
- scsi_normalize_sense(sensebuf, SCSI_SENSE_BUFFERSIZE,
- &sshdr);
if (sshdr.sense_key == RECOVERED_ERROR &&
sshdr.asc == 0 && sshdr.ascq == 0x1d)
cmd_result &= ~SAM_STAT_CHECK_CONDITION;
#include <linux/module.h>
#include <linux/kernel.h>
+#include <linux/sched/signal.h>
#include <linux/mm.h>
#include <linux/pci.h>
#include <linux/errno.h>
#include <linux/mutex.h>
#include <linux/pm_runtime.h>
#include <linux/netdevice.h>
+#include <linux/sched/signal.h>
#include <linux/sysfs.h>
#include "base.h"
if (d_really_is_positive(dentry)) {
struct kstat stat;
struct path p = {.mnt = parent.mnt, .dentry = dentry};
- err = vfs_getattr(&p, &stat);
+ err = vfs_getattr(&p, &stat, STATX_TYPE | STATX_MODE,
+ AT_STATX_SYNC_AS_STAT);
if (!err && dev_mynode(dev, d_inode(dentry), &stat)) {
struct iattr newattrs;
/*
queue_work(pm_wq, &genpd->power_off_work);
}
+/**
+ * genpd_power_off - Remove power from a given PM domain.
+ * @genpd: PM domain to power down.
+ * @one_dev_on: If invoked from genpd's ->runtime_suspend|resume() callback, the
+ * RPM status of the releated device is in an intermediate state, not yet turned
+ * into RPM_SUSPENDED. This means genpd_power_off() must allow one device to not
+ * be RPM_SUSPENDED, while it tries to power off the PM domain.
+ *
+ * If all of the @genpd's devices have been suspended and all of its subdomains
+ * have been powered down, remove power from @genpd.
+ */
+static int genpd_power_off(struct generic_pm_domain *genpd, bool one_dev_on,
+ unsigned int depth)
+{
+ struct pm_domain_data *pdd;
+ struct gpd_link *link;
+ unsigned int not_suspended = 0;
+
+ /*
+ * Do not try to power off the domain in the following situations:
+ * (1) The domain is already in the "power off" state.
+ * (2) System suspend is in progress.
+ */
+ if (genpd->status == GPD_STATE_POWER_OFF
+ || genpd->prepared_count > 0)
+ return 0;
+
+ if (atomic_read(&genpd->sd_count) > 0)
+ return -EBUSY;
+
+ list_for_each_entry(pdd, &genpd->dev_list, list_node) {
+ enum pm_qos_flags_status stat;
+
+ stat = dev_pm_qos_flags(pdd->dev,
+ PM_QOS_FLAG_NO_POWER_OFF
+ | PM_QOS_FLAG_REMOTE_WAKEUP);
+ if (stat > PM_QOS_FLAGS_NONE)
+ return -EBUSY;
+
+ /*
+ * Do not allow PM domain to be powered off, when an IRQ safe
+ * device is part of a non-IRQ safe domain.
+ */
+ if (!pm_runtime_suspended(pdd->dev) ||
+ irq_safe_dev_in_no_sleep_domain(pdd->dev, genpd))
+ not_suspended++;
+ }
+
+ if (not_suspended > 1 || (not_suspended == 1 && !one_dev_on))
+ return -EBUSY;
+
+ if (genpd->gov && genpd->gov->power_down_ok) {
+ if (!genpd->gov->power_down_ok(&genpd->domain))
+ return -EAGAIN;
+ }
+
+ if (genpd->power_off) {
+ int ret;
+
+ if (atomic_read(&genpd->sd_count) > 0)
+ return -EBUSY;
+
+ /*
+ * If sd_count > 0 at this point, one of the subdomains hasn't
+ * managed to call genpd_power_on() for the master yet after
+ * incrementing it. In that case genpd_power_on() will wait
+ * for us to drop the lock, so we can call .power_off() and let
+ * the genpd_power_on() restore power for us (this shouldn't
+ * happen very often).
+ */
+ ret = _genpd_power_off(genpd, true);
+ if (ret)
+ return ret;
+ }
+
+ genpd->status = GPD_STATE_POWER_OFF;
+
+ list_for_each_entry(link, &genpd->slave_links, slave_node) {
+ genpd_sd_counter_dec(link->master);
+ genpd_lock_nested(link->master, depth + 1);
+ genpd_power_off(link->master, false, depth + 1);
+ genpd_unlock(link->master);
+ }
+
+ return 0;
+}
+
/**
* genpd_power_on - Restore power to a given PM domain and its masters.
* @genpd: PM domain to power up.
&genpd->slave_links,
slave_node) {
genpd_sd_counter_dec(link->master);
- genpd_queue_power_off_work(link->master);
+ genpd_lock_nested(link->master, depth + 1);
+ genpd_power_off(link->master, false, depth + 1);
+ genpd_unlock(link->master);
}
return ret;
return NOTIFY_DONE;
}
-/**
- * genpd_power_off - Remove power from a given PM domain.
- * @genpd: PM domain to power down.
- * @is_async: PM domain is powered down from a scheduled work
- *
- * If all of the @genpd's devices have been suspended and all of its subdomains
- * have been powered down, remove power from @genpd.
- */
-static int genpd_power_off(struct generic_pm_domain *genpd, bool is_async)
-{
- struct pm_domain_data *pdd;
- struct gpd_link *link;
- unsigned int not_suspended = 0;
-
- /*
- * Do not try to power off the domain in the following situations:
- * (1) The domain is already in the "power off" state.
- * (2) System suspend is in progress.
- */
- if (genpd->status == GPD_STATE_POWER_OFF
- || genpd->prepared_count > 0)
- return 0;
-
- if (atomic_read(&genpd->sd_count) > 0)
- return -EBUSY;
-
- list_for_each_entry(pdd, &genpd->dev_list, list_node) {
- enum pm_qos_flags_status stat;
-
- stat = dev_pm_qos_flags(pdd->dev,
- PM_QOS_FLAG_NO_POWER_OFF
- | PM_QOS_FLAG_REMOTE_WAKEUP);
- if (stat > PM_QOS_FLAGS_NONE)
- return -EBUSY;
-
- /*
- * Do not allow PM domain to be powered off, when an IRQ safe
- * device is part of a non-IRQ safe domain.
- */
- if (!pm_runtime_suspended(pdd->dev) ||
- irq_safe_dev_in_no_sleep_domain(pdd->dev, genpd))
- not_suspended++;
- }
-
- if (not_suspended > 1 || (not_suspended == 1 && is_async))
- return -EBUSY;
-
- if (genpd->gov && genpd->gov->power_down_ok) {
- if (!genpd->gov->power_down_ok(&genpd->domain))
- return -EAGAIN;
- }
-
- if (genpd->power_off) {
- int ret;
-
- if (atomic_read(&genpd->sd_count) > 0)
- return -EBUSY;
-
- /*
- * If sd_count > 0 at this point, one of the subdomains hasn't
- * managed to call genpd_power_on() for the master yet after
- * incrementing it. In that case genpd_power_on() will wait
- * for us to drop the lock, so we can call .power_off() and let
- * the genpd_power_on() restore power for us (this shouldn't
- * happen very often).
- */
- ret = _genpd_power_off(genpd, true);
- if (ret)
- return ret;
- }
-
- genpd->status = GPD_STATE_POWER_OFF;
-
- list_for_each_entry(link, &genpd->slave_links, slave_node) {
- genpd_sd_counter_dec(link->master);
- genpd_queue_power_off_work(link->master);
- }
-
- return 0;
-}
-
/**
* genpd_power_off_work_fn - Power off PM domain whose subdomain count is 0.
* @work: Work structure used for scheduling the execution of this function.
genpd = container_of(work, struct generic_pm_domain, power_off_work);
genpd_lock(genpd);
- genpd_power_off(genpd, true);
+ genpd_power_off(genpd, false, 0);
genpd_unlock(genpd);
}
return 0;
genpd_lock(genpd);
- genpd_power_off(genpd, false);
+ genpd_power_off(genpd, true, 0);
genpd_unlock(genpd);
return 0;
if (!pm_runtime_is_irq_safe(dev) ||
(pm_runtime_is_irq_safe(dev) && genpd_is_irq_safe(genpd))) {
genpd_lock(genpd);
- genpd_power_off(genpd, 0);
+ genpd_power_off(genpd, true, 0);
genpd_unlock(genpd);
}
#include <linux/pm_wakeirq.h>
#include <linux/interrupt.h>
#include <linux/sched.h>
+#include <linux/sched/debug.h>
#include <linux/async.h>
#include <linux/suspend.h>
#include <trace/events/power.h>
* The caller needs to ensure that opp_table (and hence the regulator)
* isn't freed, while we are executing this routine.
*/
- for (i = 0; reg = regulators[i], i < count; i++) {
+ for (i = 0; i < count; i++) {
+ reg = regulators[i];
ret = regulator_set_voltage_time(reg, uV[i].min, uV[i].max);
if (ret > 0)
latency_ns += ret * 1000;
*
* This QoS design is best effort based. Dependents register their QoS needs.
* Watchers register to keep track of the current QoS needs of the system.
- * Watchers can register different types of notification callbacks:
- * . a per-device notification callback using the dev_pm_qos_*_notifier API.
- * The notification chain data is stored in the per-device constraint
- * data struct.
- * . a system-wide notification callback using the dev_pm_qos_*_global_notifier
- * API. The notification chain data is stored in a static variable.
+ * Watchers can register a per-device notification callback using the
+ * dev_pm_qos_*_notifier API. The notification chain data is stored in the
+ * per-device constraint data struct.
*
* Note about the per-device constraint data struct allocation:
* . The per-device constraints data struct ptr is tored into the device
static DEFINE_MUTEX(dev_pm_qos_mtx);
static DEFINE_MUTEX(dev_pm_qos_sysfs_mtx);
-static BLOCKING_NOTIFIER_HEAD(dev_pm_notifiers);
-
/**
* __dev_pm_qos_flags - Check PM QoS flags for a given device.
* @dev: Device to check the PM QoS flags for.
{
lockdep_assert_held(&dev->power.lock);
- return IS_ERR_OR_NULL(dev->power.qos) ?
- 0 : pm_qos_read_value(&dev->power.qos->resume_latency);
+ return dev_pm_qos_raw_read_value(dev);
}
/**
* @value: Value to assign to the QoS request.
*
* Internal function to update the constraints list using the PM QoS core
- * code and if needed call the per-device and the global notification
- * callbacks
+ * code and if needed call the per-device callbacks.
*/
static int apply_constraint(struct dev_pm_qos_request *req,
enum pm_qos_req_action action, s32 value)
case DEV_PM_QOS_RESUME_LATENCY:
ret = pm_qos_update_target(&qos->resume_latency,
&req->data.pnode, action, value);
- if (ret) {
- value = pm_qos_read_value(&qos->resume_latency);
- blocking_notifier_call_chain(&dev_pm_notifiers,
- (unsigned long)value,
- req);
- }
break;
case DEV_PM_QOS_LATENCY_TOLERANCE:
ret = pm_qos_update_target(&qos->latency_tolerance,
}
EXPORT_SYMBOL_GPL(dev_pm_qos_remove_notifier);
-/**
- * dev_pm_qos_add_global_notifier - sets notification entry for changes to
- * target value of the PM QoS constraints for any device
- *
- * @notifier: notifier block managed by caller.
- *
- * Will register the notifier into a notification chain that gets called
- * upon changes to the target value for any device.
- */
-int dev_pm_qos_add_global_notifier(struct notifier_block *notifier)
-{
- return blocking_notifier_chain_register(&dev_pm_notifiers, notifier);
-}
-EXPORT_SYMBOL_GPL(dev_pm_qos_add_global_notifier);
-
-/**
- * dev_pm_qos_remove_global_notifier - deletes notification for changes to
- * target value of PM QoS constraints for any device
- *
- * @notifier: notifier block to be removed.
- *
- * Will remove the notifier from the notification chain that gets called
- * upon changes to the target value for any device.
- */
-int dev_pm_qos_remove_global_notifier(struct notifier_block *notifier)
-{
- return blocking_notifier_chain_unregister(&dev_pm_notifiers, notifier);
-}
-EXPORT_SYMBOL_GPL(dev_pm_qos_remove_global_notifier);
-
/**
* dev_pm_qos_add_ancestor_request - Add PM QoS request for device's ancestor.
* @dev: Device whose ancestor to add the request for.
* This file is released under the GPLv2.
*/
-#include <linux/sched.h>
+#include <linux/sched/mm.h>
#include <linux/export.h>
#include <linux/pm_runtime.h>
#include <linux/pm_wakeirq.h>
#include <linux/device.h>
#include <linux/slab.h>
-#include <linux/sched.h>
+#include <linux/sched/signal.h>
#include <linux/capability.h>
#include <linux/export.h>
#include <linux/suspend.h>
pci_unmap_single(h->pdev, temp64.val, chain_sg->Len, PCI_DMA_TODEVICE);
}
-static void cciss_map_sg_chain_block(ctlr_info_t *h, CommandList_struct *c,
+static int cciss_map_sg_chain_block(ctlr_info_t *h, CommandList_struct *c,
SGDescriptor_struct *chain_block, int len)
{
SGDescriptor_struct *chain_sg;
chain_sg->Len = len;
temp64.val = pci_map_single(h->pdev, chain_block, len,
PCI_DMA_TODEVICE);
+ if (dma_mapping_error(&h->pdev->dev, temp64.val)) {
+ dev_warn(&h->pdev->dev,
+ "%s: error mapping chain block for DMA\n",
+ __func__);
+ return -1;
+ }
chain_sg->Addr.lower = temp64.val32.lower;
chain_sg->Addr.upper = temp64.val32.upper;
+
+ return 0;
}
#include "cciss_scsi.c" /* For SCSI tape support */
temp64.val = (__u64) pci_map_page(h->pdev, sg_page(&tmp_sg[i]),
tmp_sg[i].offset,
tmp_sg[i].length, dir);
+ if (dma_mapping_error(&h->pdev->dev, temp64.val)) {
+ dev_warn(&h->pdev->dev,
+ "%s: error mapping page for DMA\n", __func__);
+ creq->errors = make_status_bytes(SAM_STAT_GOOD,
+ 0, DRIVER_OK,
+ DID_SOFT_ERROR);
+ cmd_free(h, c);
+ return;
+ }
curr_sg[sg_index].Addr.lower = temp64.val32.lower;
curr_sg[sg_index].Addr.upper = temp64.val32.upper;
curr_sg[sg_index].Ext = 0; /* we are not chaining */
++sg_index;
}
- if (chained)
- cciss_map_sg_chain_block(h, c, h->cmd_sg_list[c->cmdindex],
+ if (chained) {
+ if (cciss_map_sg_chain_block(h, c, h->cmd_sg_list[c->cmdindex],
(seg - (h->max_cmd_sgentries - 1)) *
- sizeof(SGDescriptor_struct));
+ sizeof(SGDescriptor_struct))) {
+ creq->errors = make_status_bytes(SAM_STAT_GOOD,
+ 0, DRIVER_OK,
+ DID_SOFT_ERROR);
+ cmd_free(h, c);
+ return;
+ }
+ }
/* track how many SG entries we are using */
if (seg > h->maxSG)
#include <linux/compiler.h>
#include <linux/types.h>
#include <linux/list.h>
-#include <linux/sched.h>
+#include <linux/sched/signal.h>
#include <linux/bitops.h>
#include <linux/slab.h>
#include <linux/ratelimit.h>
#define __KERNEL_SYSCALLS__
#include <linux/unistd.h>
#include <linux/vmalloc.h>
+#include <linux/sched/signal.h>
#include <linux/drbd_limits.h>
#include "drbd_int.h"
int drbd_send(struct drbd_connection *connection, struct socket *sock,
void *buf, size_t size, unsigned msg_flags)
{
- struct kvec iov;
+ struct kvec iov = {.iov_base = buf, .iov_len = size};
struct msghdr msg;
int rv, sent = 0;
/* THINK if (signal_pending) return ... ? */
- iov.iov_base = buf;
- iov.iov_len = size;
-
msg.msg_name = NULL;
msg.msg_namelen = 0;
msg.msg_control = NULL;
msg.msg_controllen = 0;
msg.msg_flags = msg_flags | MSG_NOSIGNAL;
+ iov_iter_kvec(&msg.msg_iter, WRITE | ITER_KVEC, &iov, 1, size);
+
if (sock == connection->data.socket) {
rcu_read_lock();
connection->ko_count = rcu_dereference(connection->net_conf)->ko_count;
drbd_update_congested(connection);
}
do {
- rv = kernel_sendmsg(sock, &msg, &iov, 1, iov.iov_len);
+ rv = sock_sendmsg(sock, &msg);
if (rv == -EAGAIN) {
if (we_should_drop_the_connection(connection, sock))
break;
if (rv < 0)
break;
sent += rv;
- iov.iov_base += rv;
- iov.iov_len -= rv;
} while (sent < size);
if (sock == connection->data.socket)
#include <linux/memcontrol.h>
#include <linux/mm_inline.h>
#include <linux/slab.h>
+#include <uapi/linux/sched/types.h>
+#include <linux/sched/signal.h>
#include <linux/pkt_sched.h>
#define __KERNEL_SYSCALLS__
#include <linux/unistd.h>
#include <linux/module.h>
#include <linux/drbd.h>
-#include <linux/sched.h>
+#include <linux/sched/signal.h>
#include <linux/wait.h>
#include <linux/mm.h>
#include <linux/memcontrol.h>
cmd->iocb.ki_flags = IOCB_DIRECT;
if (rw == WRITE)
- ret = file->f_op->write_iter(&cmd->iocb, &iter);
+ ret = call_write_iter(file, &cmd->iocb, &iter);
else
- ret = file->f_op->read_iter(&cmd->iocb, &iter);
+ ret = call_read_iter(file, &cmd->iocb, &iter);
if (ret != -EIOCBQUEUED)
cmd->iocb.ki_complete(&cmd->iocb, ret, 0);
(info->lo_flags & LO_FLAGS_AUTOCLEAR))
lo->lo_flags ^= LO_FLAGS_AUTOCLEAR;
- if ((info->lo_flags & LO_FLAGS_PARTSCAN) &&
- !(lo->lo_flags & LO_FLAGS_PARTSCAN)) {
- lo->lo_flags |= LO_FLAGS_PARTSCAN;
- lo->lo_disk->flags &= ~GENHD_FL_NO_PART_SCAN;
- loop_reread_partitions(lo, lo->lo_device);
- }
-
lo->lo_encrypt_key_size = info->lo_encrypt_key_size;
lo->lo_init[0] = info->lo_init[0];
lo->lo_init[1] = info->lo_init[1];
exit:
blk_mq_unfreeze_queue(lo->lo_queue);
+
+ if (!err && (info->lo_flags & LO_FLAGS_PARTSCAN) &&
+ !(lo->lo_flags & LO_FLAGS_PARTSCAN)) {
+ lo->lo_flags |= LO_FLAGS_PARTSCAN;
+ lo->lo_disk->flags &= ~GENHD_FL_NO_PART_SCAN;
+ loop_reread_partitions(lo, lo->lo_device);
+ }
+
return err;
}
if (lo->lo_state != Lo_bound)
return -ENXIO;
- error = vfs_getattr(&file->f_path, &stat);
+ error = vfs_getattr(&file->f_path, &stat,
+ STATX_INO, AT_STATX_SYNC_AS_STAT);
if (error)
return error;
memset(info, 0, sizeof(*info));
/*
* Send or receive packet.
*/
-static int sock_xmit(struct nbd_device *nbd, int index, int send, void *buf,
- int size, int msg_flags)
+static int sock_xmit(struct nbd_device *nbd, int index, int send,
+ struct iov_iter *iter, int msg_flags)
{
struct socket *sock = nbd->socks[index]->sock;
int result;
struct msghdr msg;
- struct kvec iov;
unsigned long pflags = current->flags;
if (unlikely(!sock)) {
return -EINVAL;
}
+ msg.msg_iter = *iter;
+
current->flags |= PF_MEMALLOC;
do {
sock->sk->sk_allocation = GFP_NOIO | __GFP_MEMALLOC;
- iov.iov_base = buf;
- iov.iov_len = size;
msg.msg_name = NULL;
msg.msg_namelen = 0;
msg.msg_control = NULL;
msg.msg_flags = msg_flags | MSG_NOSIGNAL;
if (send)
- result = kernel_sendmsg(sock, &msg, &iov, 1, size);
+ result = sock_sendmsg(sock, &msg);
else
- result = kernel_recvmsg(sock, &msg, &iov, 1, size,
- msg.msg_flags);
+ result = sock_recvmsg(sock, &msg, msg.msg_flags);
if (result <= 0) {
if (result == 0)
result = -EPIPE; /* short read */
break;
}
- size -= result;
- buf += result;
- } while (size > 0);
+ } while (msg_data_left(&msg));
tsk_restore_flags(current, pflags, PF_MEMALLOC);
return result;
}
-static inline int sock_send_bvec(struct nbd_device *nbd, int index,
- struct bio_vec *bvec, int flags)
-{
- int result;
- void *kaddr = kmap(bvec->bv_page);
- result = sock_xmit(nbd, index, 1, kaddr + bvec->bv_offset,
- bvec->bv_len, flags);
- kunmap(bvec->bv_page);
- return result;
-}
-
/* always call with the tx_lock held */
static int nbd_send_cmd(struct nbd_device *nbd, struct nbd_cmd *cmd, int index)
{
struct request *req = blk_mq_rq_from_pdu(cmd);
int result;
- struct nbd_request request;
+ struct nbd_request request = {.magic = htonl(NBD_REQUEST_MAGIC)};
+ struct kvec iov = {.iov_base = &request, .iov_len = sizeof(request)};
+ struct iov_iter from;
unsigned long size = blk_rq_bytes(req);
struct bio *bio;
u32 type;
u32 tag = blk_mq_unique_tag(req);
+ iov_iter_kvec(&from, WRITE | ITER_KVEC, &iov, 1, sizeof(request));
+
switch (req_op(req)) {
case REQ_OP_DISCARD:
type = NBD_CMD_TRIM;
return -EIO;
}
- memset(&request, 0, sizeof(request));
- request.magic = htonl(NBD_REQUEST_MAGIC);
request.type = htonl(type);
if (type != NBD_CMD_FLUSH) {
request.from = cpu_to_be64((u64)blk_rq_pos(req) << 9);
dev_dbg(nbd_to_dev(nbd), "request %p: sending control (%s@%llu,%uB)\n",
cmd, nbdcmd_to_ascii(type),
(unsigned long long)blk_rq_pos(req) << 9, blk_rq_bytes(req));
- result = sock_xmit(nbd, index, 1, &request, sizeof(request),
+ result = sock_xmit(nbd, index, 1, &from,
(type == NBD_CMD_WRITE) ? MSG_MORE : 0);
if (result <= 0) {
dev_err_ratelimited(disk_to_dev(nbd->disk),
dev_dbg(nbd_to_dev(nbd), "request %p: sending %d bytes data\n",
cmd, bvec.bv_len);
- result = sock_send_bvec(nbd, index, &bvec, flags);
+ iov_iter_bvec(&from, ITER_BVEC | WRITE,
+ &bvec, 1, bvec.bv_len);
+ result = sock_xmit(nbd, index, 1, &from, flags);
if (result <= 0) {
dev_err(disk_to_dev(nbd->disk),
"Send data failed (result %d)\n",
return 0;
}
-static inline int sock_recv_bvec(struct nbd_device *nbd, int index,
- struct bio_vec *bvec)
-{
- int result;
- void *kaddr = kmap(bvec->bv_page);
- result = sock_xmit(nbd, index, 0, kaddr + bvec->bv_offset,
- bvec->bv_len, MSG_WAITALL);
- kunmap(bvec->bv_page);
- return result;
-}
-
/* NULL returned = something went wrong, inform userspace */
static struct nbd_cmd *nbd_read_stat(struct nbd_device *nbd, int index)
{
struct request *req = NULL;
u16 hwq;
u32 tag;
+ struct kvec iov = {.iov_base = &reply, .iov_len = sizeof(reply)};
+ struct iov_iter to;
reply.magic = 0;
- result = sock_xmit(nbd, index, 0, &reply, sizeof(reply), MSG_WAITALL);
+ iov_iter_kvec(&to, READ | ITER_KVEC, &iov, 1, sizeof(reply));
+ result = sock_xmit(nbd, index, 0, &to, MSG_WAITALL);
if (result <= 0) {
if (!test_bit(NBD_DISCONNECTED, &nbd->runtime_flags) &&
!test_bit(NBD_DISCONNECT_REQUESTED, &nbd->runtime_flags))
struct bio_vec bvec;
rq_for_each_segment(bvec, req, iter) {
- result = sock_recv_bvec(nbd, index, &bvec);
+ iov_iter_bvec(&to, ITER_BVEC | READ,
+ &bvec, 1, bvec.bv_len);
+ result = sock_xmit(nbd, index, 0, &to, MSG_WAITALL);
if (result <= 0) {
dev_err(disk_to_dev(nbd->disk), "Receive data failed (result %d)\n",
result);
static void send_disconnects(struct nbd_device *nbd)
{
- struct nbd_request request = {};
+ struct nbd_request request = {
+ .magic = htonl(NBD_REQUEST_MAGIC),
+ .type = htonl(NBD_CMD_DISC),
+ };
+ struct kvec iov = {.iov_base = &request, .iov_len = sizeof(request)};
+ struct iov_iter from;
int i, ret;
- request.magic = htonl(NBD_REQUEST_MAGIC);
- request.type = htonl(NBD_CMD_DISC);
-
for (i = 0; i < nbd->num_connections; i++) {
- ret = sock_xmit(nbd, i, 1, &request, sizeof(request), 0);
+ iov_iter_kvec(&from, WRITE | ITER_KVEC, &iov, 1, sizeof(request));
+ ret = sock_xmit(nbd, i, 1, &from, 0);
if (ret <= 0)
dev_err(disk_to_dev(nbd->disk),
"Send disconnect failed %d\n", ret);
nbd->num_connections) {
int i;
- for (i = 0; i < nbd->num_connections; i++)
+ for (i = 0; i < nbd->num_connections; i++) {
+ sockfd_put(nbd->socks[i]->sock);
kfree(nbd->socks[i]);
+ }
kfree(nbd->socks);
nbd->socks = NULL;
nbd->num_connections = 0;
the slower the port i/o. In some cases, setting
this to zero will speed up the device. (default -1)
- major You may use this parameter to overide the
+ major You may use this parameter to override the
default major number (46) that this driver
will use. Be sure to change the device
name as well.
first drive found.
- major You may use this parameter to overide the
+ major You may use this parameter to override the
default major number (45) that this driver
will use. Be sure to change the device
name as well.
the slower the port i/o. In some cases, setting
this to zero will speed up the device. (default -1)
- major You may use this parameter to overide the
+ major You may use this parameter to override the
default major number (47) that this driver
will use. Be sure to change the device
name as well.
the slower the port i/o. In some cases, setting
this to zero will speed up the device. (default -1)
- major You may use this parameter to overide the
+ major You may use this parameter to override the
default major number (97) that this driver
will use. Be sure to change the device
name as well.
the slower the port i/o. In some cases, setting
this to zero will speed up the device. (default -1)
- major You may use this parameter to overide the
+ major You may use this parameter to override the
default major number (96) that this driver
will use. Be sure to change the device
name as well.
/* Feature bits */
-#define RBD_FEATURE_LAYERING (1<<0)
-#define RBD_FEATURE_STRIPINGV2 (1<<1)
-#define RBD_FEATURE_EXCLUSIVE_LOCK (1<<2)
-#define RBD_FEATURE_DATA_POOL (1<<7)
+#define RBD_FEATURE_LAYERING (1ULL<<0)
+#define RBD_FEATURE_STRIPINGV2 (1ULL<<1)
+#define RBD_FEATURE_EXCLUSIVE_LOCK (1ULL<<2)
+#define RBD_FEATURE_DATA_POOL (1ULL<<7)
+
#define RBD_FEATURES_ALL (RBD_FEATURE_LAYERING | \
RBD_FEATURE_STRIPINGV2 | \
RBD_FEATURE_EXCLUSIVE_LOCK | \
return is_lock_owner;
}
+static ssize_t rbd_supported_features_show(struct bus_type *bus, char *buf)
+{
+ return sprintf(buf, "0x%llx\n", RBD_FEATURES_SUPPORTED);
+}
+
static BUS_ATTR(add, S_IWUSR, NULL, rbd_add);
static BUS_ATTR(remove, S_IWUSR, NULL, rbd_remove);
static BUS_ATTR(add_single_major, S_IWUSR, NULL, rbd_add_single_major);
static BUS_ATTR(remove_single_major, S_IWUSR, NULL, rbd_remove_single_major);
+static BUS_ATTR(supported_features, S_IRUGO, rbd_supported_features_show, NULL);
static struct attribute *rbd_bus_attrs[] = {
&bus_attr_add.attr,
&bus_attr_remove.attr,
&bus_attr_add_single_major.attr,
&bus_attr_remove_single_major.attr,
+ &bus_attr_supported_features.attr,
NULL,
};
#include <linux/stddef.h>
#include <linux/kernel.h>
-#include <linux/sched.h>
+#include <linux/sched/signal.h>
#include <linux/timer.h>
#include <linux/delay.h>
#include <linux/fd.h>
#include <linux/hdreg.h>
#include <linux/module.h>
#include <linux/mutex.h>
+#include <linux/interrupt.h>
#include <linux/virtio.h>
#include <linux/virtio_blk.h>
#include <linux/scatterlist.h>
#include <scsi/scsi_cmnd.h>
#include <linux/idr.h>
#include <linux/blk-mq.h>
+#include <linux/blk-mq-virtio.h>
#include <linux/numa.h>
#define PART_BITS 4
struct virtqueue **vqs;
unsigned short num_vqs;
struct virtio_device *vdev = vblk->vdev;
+ struct irq_affinity desc = { 0, };
err = virtio_cread_feature(vdev, VIRTIO_BLK_F_MQ,
struct virtio_blk_config, num_queues,
}
/* Discover virtqueues and write information to configuration. */
- err = vdev->config->find_vqs(vdev, num_vqs, vqs, callbacks, names);
+ err = vdev->config->find_vqs(vdev, num_vqs, vqs, callbacks, names,
+ &desc);
if (err)
goto out;
return 0;
}
+static int virtblk_map_queues(struct blk_mq_tag_set *set)
+{
+ struct virtio_blk *vblk = set->driver_data;
+
+ return blk_mq_virtio_map_queues(set, vblk->vdev, 0);
+}
+
static struct blk_mq_ops virtio_mq_ops = {
.queue_rq = virtio_queue_rq,
.complete = virtblk_request_done,
.init_request = virtblk_init_request,
+ .map_queues = virtblk_map_queues,
};
static unsigned int virtblk_queue_depth;
blk_queue_io_min(zram->disk->queue, PAGE_SIZE);
blk_queue_io_opt(zram->disk->queue, PAGE_SIZE);
zram->disk->queue->limits.discard_granularity = PAGE_SIZE;
+ zram->disk->queue->limits.max_sectors = SECTORS_PER_PAGE;
+ zram->disk->queue->limits.chunk_sectors = 0;
blk_queue_max_discard_sectors(zram->disk->queue, UINT_MAX);
/*
* zram_bio_discard() will clear all logical blocks if logical block
#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/interrupt.h>
-#include <linux/sched.h>
+#include <linux/sched/signal.h>
#include <linux/slab.h>
#include <linux/errno.h>
#include <linux/mutex.h>
#include <linux/spinlock.h>
#include <linux/sysctl.h>
#include <linux/wait.h>
+#include <linux/sched/signal.h>
#include <linux/bcd.h>
#include <linux/seq_file.h>
#include <linux/bitops.h>
#include <linux/hw_random.h>
#include <linux/kernel.h>
#include <linux/kthread.h>
+#include <linux/sched/signal.h>
#include <linux/miscdevice.h>
#include <linux/module.h>
#include <linux/random.h>
#include <linux/ctype.h>
#include <linux/delay.h>
#include <linux/atomic.h>
+#include <linux/sched/signal.h>
#ifdef CONFIG_X86
/*
#include <linux/errno.h>
#include <linux/kernel.h>
#include <linux/major.h>
-#include <linux/sched.h>
+#include <linux/sched/signal.h>
#include <linux/slab.h>
#include <linux/fcntl.h>
#include <linux/delay.h>
#include <linux/module.h>
#include <linux/kernel.h>
-#include <linux/sched.h>
+#include <linux/sched/signal.h>
#include <linux/interrupt.h>
#include <linux/time.h>
#include <linux/timer.h>
#include <linux/module.h>
#include <linux/init.h>
-#include <linux/sched.h>
+#include <linux/sched/signal.h>
#include <linux/device.h>
#include <linux/ioctl.h>
#include <linux/parport.h>
*/
static int random_write_wakeup_bits = 28 * OUTPUT_POOL_WORDS;
-/*
- * The minimum number of seconds between urandom pool reseeding. We
- * do this to limit the amount of entropy that can be drained from the
- * input pool even if there are heavy demands on /dev/urandom.
- */
-static int random_min_urandom_seed = 60;
-
/*
* Originally, we used a primitive polynomial of degree .poolwords
* over GF(2). The taps for various sizes are defined below. They
*/
static DECLARE_WAIT_QUEUE_HEAD(random_read_wait);
static DECLARE_WAIT_QUEUE_HEAD(random_write_wait);
-static DECLARE_WAIT_QUEUE_HEAD(urandom_init_wait);
static struct fasync_struct *fasync;
static DEFINE_SPINLOCK(random_ready_list_lock);
int entropy_count;
int entropy_total;
unsigned int initialized:1;
- unsigned int limit:1;
unsigned int last_data_init:1;
__u8 last_data[EXTRACT_SIZE];
};
static struct entropy_store input_pool = {
.poolinfo = &poolinfo_table[0],
.name = "input",
- .limit = 1,
.lock = __SPIN_LOCK_UNLOCKED(input_pool.lock),
.pool = input_pool_data
};
static struct entropy_store blocking_pool = {
.poolinfo = &poolinfo_table[1],
.name = "blocking",
- .limit = 1,
.pull = &input_pool,
.lock = __SPIN_LOCK_UNLOCKED(blocking_pool.lock),
.pool = blocking_pool_data,
spin_unlock_irqrestore(&primary_crng.lock, flags);
}
-static inline void maybe_reseed_primary_crng(void)
-{
- if (crng_init > 2 &&
- time_after(jiffies, primary_crng.init_time + CRNG_RESEED_INTERVAL))
- crng_reseed(&primary_crng, &input_pool);
-}
-
static inline void crng_wait_ready(void)
{
wait_event_interruptible(crng_init_wait, crng_ready());
r->entropy_count > r->poolinfo->poolfracbits)
return;
- if (r->limit == 0 && random_min_urandom_seed) {
- unsigned long now = jiffies;
-
- if (time_before(now,
- r->last_pulled + random_min_urandom_seed * HZ))
- return;
- r->last_pulled = now;
- }
-
_xfer_secondary_pool(r, nbytes);
}
{
__u32 tmp[OUTPUT_POOL_WORDS];
- /* For /dev/random's pool, always leave two wakeups' worth */
- int rsvd_bytes = r->limit ? 0 : random_read_wakeup_bits / 4;
int bytes = nbytes;
/* pull at least as much as a wakeup */
trace_xfer_secondary_pool(r->name, bytes * 8, nbytes * 8,
ENTROPY_BITS(r), ENTROPY_BITS(r->pull));
bytes = extract_entropy(r->pull, tmp, bytes,
- random_read_wakeup_bits / 8, rsvd_bytes);
+ random_read_wakeup_bits / 8, 0);
mix_pool_bytes(r, tmp, bytes);
credit_entropy_bits(r, bytes*8);
}
static size_t account(struct entropy_store *r, size_t nbytes, int min,
int reserved)
{
- int entropy_count, orig;
+ int entropy_count, orig, have_bytes;
size_t ibytes, nfrac;
BUG_ON(r->entropy_count > r->poolinfo->poolfracbits);
retry:
entropy_count = orig = ACCESS_ONCE(r->entropy_count);
ibytes = nbytes;
- /* If limited, never pull more than available */
- if (r->limit) {
- int have_bytes = entropy_count >> (ENTROPY_SHIFT + 3);
+ /* never pull more than available */
+ have_bytes = entropy_count >> (ENTROPY_SHIFT + 3);
- if ((have_bytes -= reserved) < 0)
- have_bytes = 0;
- ibytes = min_t(size_t, ibytes, have_bytes);
- }
+ if ((have_bytes -= reserved) < 0)
+ have_bytes = 0;
+ ibytes = min_t(size_t, ibytes, have_bytes);
if (ibytes < min)
ibytes = 0;
static int min_read_thresh = 8, min_write_thresh;
static int max_read_thresh = OUTPUT_POOL_WORDS * 32;
static int max_write_thresh = INPUT_POOL_WORDS * 32;
+static int random_min_urandom_seed = 60;
static char sysctl_bootid[16];
/*
};
#endif /* CONFIG_SYSCTL */
-static u32 random_int_secret[MD5_MESSAGE_BYTES / 4] ____cacheline_aligned;
-
-int random_int_secret_init(void)
-{
- get_random_bytes(random_int_secret, sizeof(random_int_secret));
- return 0;
-}
-
-static DEFINE_PER_CPU(__u32 [MD5_DIGEST_WORDS], get_random_int_hash)
- __aligned(sizeof(unsigned long));
+struct batched_entropy {
+ union {
+ u64 entropy_u64[CHACHA20_BLOCK_SIZE / sizeof(u64)];
+ u32 entropy_u32[CHACHA20_BLOCK_SIZE / sizeof(u32)];
+ };
+ unsigned int position;
+};
/*
- * Get a random word for internal kernel use only. Similar to urandom but
- * with the goal of minimal entropy pool depletion. As a result, the random
- * value is not cryptographically secure but for several uses the cost of
- * depleting entropy is too high
+ * Get a random word for internal kernel use only. The quality of the random
+ * number is either as good as RDRAND or as good as /dev/urandom, with the
+ * goal of being quite fast and not depleting entropy.
*/
-unsigned int get_random_int(void)
+static DEFINE_PER_CPU(struct batched_entropy, batched_entropy_u64);
+u64 get_random_u64(void)
{
- __u32 *hash;
- unsigned int ret;
+ u64 ret;
+ struct batched_entropy *batch;
- if (arch_get_random_int(&ret))
+#if BITS_PER_LONG == 64
+ if (arch_get_random_long((unsigned long *)&ret))
return ret;
+#else
+ if (arch_get_random_long((unsigned long *)&ret) &&
+ arch_get_random_long((unsigned long *)&ret + 1))
+ return ret;
+#endif
- hash = get_cpu_var(get_random_int_hash);
-
- hash[0] += current->pid + jiffies + random_get_entropy();
- md5_transform(hash, random_int_secret);
- ret = hash[0];
- put_cpu_var(get_random_int_hash);
-
+ batch = &get_cpu_var(batched_entropy_u64);
+ if (batch->position % ARRAY_SIZE(batch->entropy_u64) == 0) {
+ extract_crng((u8 *)batch->entropy_u64);
+ batch->position = 0;
+ }
+ ret = batch->entropy_u64[batch->position++];
+ put_cpu_var(batched_entropy_u64);
return ret;
}
-EXPORT_SYMBOL(get_random_int);
+EXPORT_SYMBOL(get_random_u64);
-/*
- * Same as get_random_int(), but returns unsigned long.
- */
-unsigned long get_random_long(void)
+static DEFINE_PER_CPU(struct batched_entropy, batched_entropy_u32);
+u32 get_random_u32(void)
{
- __u32 *hash;
- unsigned long ret;
+ u32 ret;
+ struct batched_entropy *batch;
- if (arch_get_random_long(&ret))
+ if (arch_get_random_int(&ret))
return ret;
- hash = get_cpu_var(get_random_int_hash);
-
- hash[0] += current->pid + jiffies + random_get_entropy();
- md5_transform(hash, random_int_secret);
- ret = *(unsigned long *)hash;
- put_cpu_var(get_random_int_hash);
-
+ batch = &get_cpu_var(batched_entropy_u32);
+ if (batch->position % ARRAY_SIZE(batch->entropy_u32) == 0) {
+ extract_crng((u8 *)batch->entropy_u32);
+ batch->position = 0;
+ }
+ ret = batch->entropy_u32[batch->position++];
+ put_cpu_var(batched_entropy_u32);
return ret;
}
-EXPORT_SYMBOL(get_random_long);
+EXPORT_SYMBOL(get_random_u32);
/**
* randomize_page - Generate a random, page aligned address
#include <linux/proc_fs.h>
#include <linux/seq_file.h>
#include <linux/spinlock.h>
-#include <linux/sched.h>
+#include <linux/sched/signal.h>
#include <linux/sysctl.h>
#include <linux/wait.h>
#include <linux/bcd.h>
*/
#include <linux/interrupt.h>
-#include <linux/sched.h>
+#include <linux/sched/signal.h>
#include <linux/device.h>
#include <linux/poll.h>
#include <linux/init.h>
*/
#include <linux/interrupt.h>
-#include <linux/sched.h>
+#include <linux/sched/signal.h>
#include <linux/slab.h>
#include <asm/byteorder.h>
#include <asm/sn/sn_sal.h>
{
struct port *port;
struct scatterlist sg[1];
+ void *data;
+ int ret;
if (unlikely(early_put_chars))
return early_put_chars(vtermno, buf, count);
if (!port)
return -EPIPE;
- sg_init_one(sg, buf, count);
- return __send_to_port(port, sg, 1, count, (void *)buf, false);
+ data = kmemdup(buf, count, GFP_ATOMIC);
+ if (!data)
+ return -ENOMEM;
+
+ sg_init_one(sg, data, count);
+ ret = __send_to_port(port, sg, 1, count, data, false);
+ kfree(data);
+ return ret;
}
/*
/* Find the queues. */
err = portdev->vdev->config->find_vqs(portdev->vdev, nr_queues, vqs,
io_callbacks,
- (const char **)io_names);
+ (const char **)io_names, NULL);
if (err)
goto free;
},
};
+static struct clk_mux gxbb_sar_adc_clk_sel = {
+ .reg = (void *)HHI_SAR_CLK_CNTL,
+ .mask = 0x3,
+ .shift = 9,
+ .lock = &clk_lock,
+ .hw.init = &(struct clk_init_data){
+ .name = "sar_adc_clk_sel",
+ .ops = &clk_mux_ops,
+ /* NOTE: The datasheet doesn't list the parents for bit 10 */
+ .parent_names = (const char *[]){ "xtal", "clk81", },
+ .num_parents = 2,
+ },
+};
+
+static struct clk_divider gxbb_sar_adc_clk_div = {
+ .reg = (void *)HHI_SAR_CLK_CNTL,
+ .shift = 0,
+ .width = 8,
+ .lock = &clk_lock,
+ .hw.init = &(struct clk_init_data){
+ .name = "sar_adc_clk_div",
+ .ops = &clk_divider_ops,
+ .parent_names = (const char *[]){ "sar_adc_clk_sel" },
+ .num_parents = 1,
+ },
+};
+
+static struct clk_gate gxbb_sar_adc_clk = {
+ .reg = (void *)HHI_SAR_CLK_CNTL,
+ .bit_idx = 8,
+ .lock = &clk_lock,
+ .hw.init = &(struct clk_init_data){
+ .name = "sar_adc_clk",
+ .ops = &clk_gate_ops,
+ .parent_names = (const char *[]){ "sar_adc_clk_div" },
+ .num_parents = 1,
+ .flags = CLK_SET_RATE_PARENT,
+ },
+};
+
/* Everything Else (EE) domain gates */
static MESON_GATE(gxbb_ddr, HHI_GCLK_MPEG0, 0);
static MESON_GATE(gxbb_dos, HHI_GCLK_MPEG0, 1);
[CLKID_SD_EMMC_A] = &gxbb_emmc_a.hw,
[CLKID_SD_EMMC_B] = &gxbb_emmc_b.hw,
[CLKID_SD_EMMC_C] = &gxbb_emmc_c.hw,
+ [CLKID_SAR_ADC_CLK] = &gxbb_sar_adc_clk.hw,
+ [CLKID_SAR_ADC_SEL] = &gxbb_sar_adc_clk_sel.hw,
+ [CLKID_SAR_ADC_DIV] = &gxbb_sar_adc_clk_div.hw,
},
.num = NR_CLKS,
};
&gxbb_emmc_a,
&gxbb_emmc_b,
&gxbb_emmc_c,
+ &gxbb_sar_adc_clk,
};
static int gxbb_clkc_probe(struct platform_device *pdev)
gxbb_mpeg_clk_sel.reg = clk_base + (u64)gxbb_mpeg_clk_sel.reg;
gxbb_mpeg_clk_div.reg = clk_base + (u64)gxbb_mpeg_clk_div.reg;
+ /* Populate the base address for the SAR ADC clks */
+ gxbb_sar_adc_clk_sel.reg = clk_base + (u64)gxbb_sar_adc_clk_sel.reg;
+ gxbb_sar_adc_clk_div.reg = clk_base + (u64)gxbb_sar_adc_clk_div.reg;
+
/* Populate base address for gates */
for (i = 0; i < ARRAY_SIZE(gxbb_clk_gates); i++)
gxbb_clk_gates[i]->reg = clk_base +
#define CLKID_PERIPHS 20
#define CLKID_SPICC 21
/* CLKID_I2C */
-#define CLKID_SAR_ADC 23
+/* #define CLKID_SAR_ADC */
#define CLKID_SMART_CARD 24
#define CLKID_RNG0 25
#define CLKID_UART0 26
#define CLKID_ASSIST_MISC 33
/* CLKID_SPI */
#define CLKID_I2S_SPDIF 35
-#define CLKID_ETH 36
+/* CLKID_ETH */
#define CLKID_DEMUX 37
#define CLKID_AIU_GLUE 38
#define CLKID_IEC958 39
#define CLKID_AHB_DATA_BUS 60
#define CLKID_AHB_CTRL_BUS 61
#define CLKID_HDMI_INTR_SYNC 62
-#define CLKID_HDMI_PCLK 63
+/* CLKID_HDMI_PCLK */
/* CLKID_USB1_DDR_BRIDGE */
/* CLKID_USB0_DDR_BRIDGE */
#define CLKID_MMC_PCLK 66
#define CLKID_DVIN 67
#define CLKID_UART2 68
-#define CLKID_SANA 69
+/* #define CLKID_SANA */
#define CLKID_VPU_INTR 70
#define CLKID_SEC_AHB_AHB3_BRIDGE 71
#define CLKID_CLK81_A53 72
#define CLKID_VCLK2_VENCI1 74
#define CLKID_VCLK2_VENCP0 75
#define CLKID_VCLK2_VENCP1 76
-#define CLKID_GCLK_VENCI_INT0 77
+/* CLKID_GCLK_VENCI_INT0 */
#define CLKID_GCLK_VENCI_INT 78
#define CLKID_DAC_CLK 79
#define CLKID_AOCLK_GATE 80
/* CLKID_SD_EMMC_A */
/* CLKID_SD_EMMC_B */
/* CLKID_SD_EMMC_C */
+/* CLKID_SAR_ADC_CLK */
+/* CLKID_SAR_ADC_SEL */
+#define CLKID_SAR_ADC_DIV 99
-#define NR_CLKS 97
+#define NR_CLKS 100
/* include the CLKIDs that have been made part of the stable DT binding */
#include <dt-bindings/clock/gxbb-clkc.h>
#include <linux/of_address.h>
#include <linux/io.h>
#include <linux/slab.h>
+#include <linux/sched/clock.h>
#include <linux/sched_clock.h>
#include <linux/acpi.h>
#include <linux/clockchips.h>
#include <linux/of_address.h>
#include <linux/of_irq.h>
+#include <linux/sched/clock.h>
#include <linux/sched_clock.h>
#include <clocksource/pxa.h>
#include <linux/interrupt.h>
#include <linux/irq.h>
#include <linux/irqreturn.h>
+#include <linux/sched/clock.h>
#include <linux/sched_clock.h>
#include <linux/of.h>
#include <linux/of_address.h>
return 0;
}
+module_param(off, int, 0444);
core_initcall(cpufreq_core_init);
#include <linux/export.h>
#include <linux/kernel_stat.h>
-#include <linux/sched.h>
#include <linux/slab.h>
#include "cpufreq_governor.h"
#include <linux/atomic.h>
#include <linux/irq_work.h>
#include <linux/cpufreq.h>
+#include <linux/sched/cpufreq.h>
#include <linux/kernel_stat.h>
#include <linux/module.h>
#include <linux/mutex.h>
#include <linux/percpu-defs.h>
#include <linux/slab.h>
#include <linux/tick.h>
+#include <linux/sched/cpufreq.h>
#include "cpufreq_ondemand.h"
#include <linux/hrtimer.h>
#include <linux/tick.h>
#include <linux/slab.h>
-#include <linux/sched.h>
+#include <linux/sched/cpufreq.h>
#include <linux/list.h>
#include <linux/cpu.h>
#include <linux/cpufreq.h>
#define INTEL_CPUFREQ_TRANSITION_LATENCY 20000
-#define ATOM_RATIOS 0x66a
-#define ATOM_VIDS 0x66b
-#define ATOM_TURBO_RATIOS 0x66c
-#define ATOM_TURBO_VIDS 0x66d
-
#ifdef CONFIG_ACPI
#include <acpi/processor.h>
#include <acpi/cppc_acpi.h>
static bool acpi_ppc;
#endif
-static struct perf_limits performance_limits = {
- .no_turbo = 0,
- .turbo_disabled = 0,
- .max_perf_pct = 100,
- .max_perf = int_ext_tofp(1),
- .min_perf_pct = 100,
- .min_perf = int_ext_tofp(1),
- .max_policy_pct = 100,
- .max_sysfs_pct = 100,
- .min_policy_pct = 0,
- .min_sysfs_pct = 0,
-};
+static struct perf_limits performance_limits;
+static struct perf_limits powersave_limits;
+static struct perf_limits *limits;
-static struct perf_limits powersave_limits = {
- .no_turbo = 0,
- .turbo_disabled = 0,
- .max_perf_pct = 100,
- .max_perf = int_ext_tofp(1),
- .min_perf_pct = 0,
- .min_perf = 0,
- .max_policy_pct = 100,
- .max_sysfs_pct = 100,
- .min_policy_pct = 0,
- .min_sysfs_pct = 0,
-};
+static void intel_pstate_init_limits(struct perf_limits *limits)
+{
+ memset(limits, 0, sizeof(*limits));
+ limits->max_perf_pct = 100;
+ limits->max_perf = int_ext_tofp(1);
+ limits->max_policy_pct = 100;
+ limits->max_sysfs_pct = 100;
+}
-#ifdef CONFIG_CPU_FREQ_DEFAULT_GOV_PERFORMANCE
-static struct perf_limits *limits = &performance_limits;
-#else
-static struct perf_limits *limits = &powersave_limits;
-#endif
+static void intel_pstate_set_performance_limits(struct perf_limits *limits)
+{
+ intel_pstate_init_limits(limits);
+ limits->min_perf_pct = 100;
+ limits->min_perf = int_ext_tofp(1);
+ limits->min_sysfs_pct = 100;
+}
static DEFINE_MUTEX(intel_pstate_driver_lock);
static DEFINE_MUTEX(intel_pstate_limits_lock);
}
static void intel_pstate_update_policies(void)
+ __releases(&intel_pstate_limits_lock)
+ __acquires(&intel_pstate_limits_lock)
{
+ struct perf_limits *saved_limits = limits;
int cpu;
+ mutex_unlock(&intel_pstate_limits_lock);
+
for_each_possible_cpu(cpu)
cpufreq_update_policy(cpu);
+
+ mutex_lock(&intel_pstate_limits_lock);
+
+ limits = saved_limits;
}
/************************** debugfs begin ************************/
limits->no_turbo = clamp_t(int, input, 0, 1);
- mutex_unlock(&intel_pstate_limits_lock);
-
intel_pstate_update_policies();
+ mutex_unlock(&intel_pstate_limits_lock);
+
mutex_unlock(&intel_pstate_driver_lock);
return count;
limits->max_perf_pct);
limits->max_perf = div_ext_fp(limits->max_perf_pct, 100);
- mutex_unlock(&intel_pstate_limits_lock);
-
intel_pstate_update_policies();
+ mutex_unlock(&intel_pstate_limits_lock);
+
mutex_unlock(&intel_pstate_driver_lock);
return count;
limits->min_perf_pct);
limits->min_perf = div_ext_fp(limits->min_perf_pct, 100);
- mutex_unlock(&intel_pstate_limits_lock);
-
intel_pstate_update_policies();
+ mutex_unlock(&intel_pstate_limits_lock);
+
mutex_unlock(&intel_pstate_driver_lock);
return count;
{
u64 value;
- rdmsrl(ATOM_RATIOS, value);
+ rdmsrl(MSR_ATOM_CORE_RATIOS, value);
return (value >> 8) & 0x7F;
}
{
u64 value;
- rdmsrl(ATOM_RATIOS, value);
+ rdmsrl(MSR_ATOM_CORE_RATIOS, value);
return (value >> 16) & 0x7F;
}
{
u64 value;
- rdmsrl(ATOM_TURBO_RATIOS, value);
+ rdmsrl(MSR_ATOM_CORE_TURBO_RATIOS, value);
return value & 0x7F;
}
{
u64 value;
- rdmsrl(ATOM_VIDS, value);
+ rdmsrl(MSR_ATOM_CORE_VIDS, value);
cpudata->vid.min = int_tofp((value >> 8) & 0x7f);
cpudata->vid.max = int_tofp((value >> 16) & 0x7f);
cpudata->vid.ratio = div_fp(
int_tofp(cpudata->pstate.max_pstate -
cpudata->pstate.min_pstate));
- rdmsrl(ATOM_TURBO_VIDS, value);
+ rdmsrl(MSR_ATOM_CORE_TURBO_VIDS, value);
cpudata->vid.turbo = value & 0x7f;
}
intel_pstate_get_min_max(cpu, &min_perf, &max_perf);
pstate = clamp_t(int, pstate, min_perf, max_perf);
- trace_cpu_frequency(pstate * cpu->pstate.scaling, cpu->cpu);
return pstate;
}
static void intel_pstate_update_pstate(struct cpudata *cpu, int pstate)
{
- pstate = intel_pstate_prepare_request(cpu, pstate);
if (pstate == cpu->pstate.current_pstate)
return;
update_turbo_state();
+ target_pstate = intel_pstate_prepare_request(cpu, target_pstate);
+ trace_cpu_frequency(target_pstate * cpu->pstate.scaling, cpu->cpu);
intel_pstate_update_pstate(cpu, target_pstate);
sample = &cpu->sample;
synchronize_sched();
}
-static void intel_pstate_set_performance_limits(struct perf_limits *limits)
-{
- limits->no_turbo = 0;
- limits->turbo_disabled = 0;
- limits->max_perf_pct = 100;
- limits->max_perf = int_ext_tofp(1);
- limits->min_perf_pct = 100;
- limits->min_perf = int_ext_tofp(1);
- limits->max_policy_pct = 100;
- limits->max_sysfs_pct = 100;
- limits->min_policy_pct = 0;
- limits->min_sysfs_pct = 0;
-}
-
static void intel_pstate_update_perf_limits(struct cpufreq_policy *policy,
struct perf_limits *limits)
{
mutex_lock(&intel_pstate_limits_lock);
if (policy->policy == CPUFREQ_POLICY_PERFORMANCE) {
+ pr_debug("set performance\n");
if (!perf_limits) {
limits = &performance_limits;
perf_limits = limits;
}
- if (policy->max >= policy->cpuinfo.max_freq &&
- !limits->no_turbo) {
- pr_debug("set performance\n");
- intel_pstate_set_performance_limits(perf_limits);
- goto out;
- }
} else {
pr_debug("set powersave\n");
if (!perf_limits) {
}
intel_pstate_update_perf_limits(policy, perf_limits);
- out:
+
if (cpu->policy == CPUFREQ_POLICY_PERFORMANCE) {
/*
* NOHZ_FULL CPUs need this as the governor callback may not
unsigned int max_freq, min_freq;
max_freq = policy->cpuinfo.max_freq *
- limits->max_sysfs_pct / 100;
+ perf_limits->max_sysfs_pct / 100;
min_freq = policy->cpuinfo.max_freq *
- limits->min_sysfs_pct / 100;
+ perf_limits->min_sysfs_pct / 100;
cpufreq_verify_within_limits(policy, min_freq, max_freq);
}
cpu = all_cpu_data[policy->cpu];
- /*
- * We need sane value in the cpu->perf_limits, so inherit from global
- * perf_limits limits, which are seeded with values based on the
- * CONFIG_CPU_FREQ_DEFAULT_GOV_*, during boot up.
- */
if (per_cpu_limits)
- memcpy(cpu->perf_limits, limits, sizeof(struct perf_limits));
+ intel_pstate_init_limits(cpu->perf_limits);
policy->min = cpu->pstate.min_pstate * cpu->pstate.scaling;
policy->max = cpu->pstate.turbo_pstate * cpu->pstate.scaling;
static int intel_cpufreq_verify_policy(struct cpufreq_policy *policy)
{
struct cpudata *cpu = all_cpu_data[policy->cpu];
- struct perf_limits *perf_limits = limits;
update_turbo_state();
policy->cpuinfo.max_freq = limits->turbo_disabled ?
cpufreq_verify_within_cpu_limits(policy);
- if (per_cpu_limits)
- perf_limits = cpu->perf_limits;
-
- mutex_lock(&intel_pstate_limits_lock);
-
- intel_pstate_update_perf_limits(policy, perf_limits);
-
- mutex_unlock(&intel_pstate_limits_lock);
-
return 0;
}
wrmsrl_on_cpu(policy->cpu, MSR_IA32_PERF_CTL,
pstate_funcs.get_val(cpu, target_pstate));
}
+ freqs.new = target_pstate * cpu->pstate.scaling;
cpufreq_freq_transition_end(policy, &freqs, false);
return 0;
target_freq = intel_cpufreq_turbo_update(cpu, policy, target_freq);
target_pstate = DIV_ROUND_UP(target_freq, cpu->pstate.scaling);
+ target_pstate = intel_pstate_prepare_request(cpu, target_pstate);
intel_pstate_update_pstate(cpu, target_pstate);
- return target_freq;
+ return target_pstate * cpu->pstate.scaling;
}
static int intel_cpufreq_cpu_init(struct cpufreq_policy *policy)
{
int ret;
+ intel_pstate_init_limits(&powersave_limits);
+ intel_pstate_set_performance_limits(&performance_limits);
+ if (IS_ENABLED(CONFIG_CPU_FREQ_DEFAULT_GOV_PERFORMANCE) &&
+ intel_pstate_driver == &intel_pstate)
+ limits = &performance_limits;
+ else
+ limits = &powersave_limits;
+
ret = cpufreq_register_driver(intel_pstate_driver);
if (ret) {
intel_pstate_driver_cleanup();
#include <linux/slab.h>
#include <linux/smp.h>
-#if !defined(CONFIG_ARM)
-#include <asm/smp.h> /* for get_hard_smp_processor_id() in UP configs */
-#endif
-
/**
* struct cpu_data
* @pclk: the parent clock of cpu
cpumask_t cpus_allowed;
unsigned long clock_tick, estar;
- cpumask_copy(&cpus_allowed, tsk_cpus_allowed(current));
+ cpumask_copy(&cpus_allowed, ¤t->cpus_allowed);
set_cpus_allowed_ptr(current, cpumask_of(cpu));
clock_tick = sparc64_get_clock_tick(cpu) / 1000;
unsigned long clock_tick, divisor, old_divisor, estar;
cpumask_t cpus_allowed;
- cpumask_copy(&cpus_allowed, tsk_cpus_allowed(current));
+ cpumask_copy(&cpus_allowed, ¤t->cpus_allowed);
set_cpus_allowed_ptr(current, cpumask_of(cpu));
new_freq = clock_tick = sparc64_get_clock_tick(cpu) / 1000;
unsigned long reg;
unsigned int ret;
- cpumask_copy(&cpus_allowed, tsk_cpus_allowed(current));
+ cpumask_copy(&cpus_allowed, ¤t->cpus_allowed);
set_cpus_allowed_ptr(current, cpumask_of(cpu));
reg = read_safari_cfg();
unsigned long new_bits, new_freq, reg;
cpumask_t cpus_allowed;
- cpumask_copy(&cpus_allowed, tsk_cpus_allowed(current));
+ cpumask_copy(&cpus_allowed, ¤t->cpus_allowed);
set_cpus_allowed_ptr(current, cpumask_of(cpu));
new_freq = sparc64_get_clock_tick(cpu) / 1000;
#include <linux/kernel.h>
#include <linux/mutex.h>
#include <linux/sched.h>
+#include <linux/sched/clock.h>
#include <linux/notifier.h>
#include <linux/pm_qos.h>
#include <linux/cpu.h>
#include <linux/mutex.h>
#include <linux/module.h>
#include <linux/sched.h>
+#include <linux/sched/idle.h>
#include <linux/cpuidle.h>
#include <linux/cpumask.h>
#include <linux/tick.h>
#include <linux/hrtimer.h>
#include <linux/tick.h>
#include <linux/sched.h>
+#include <linux/sched/loadavg.h>
+#include <linux/sched/stat.h>
#include <linux/math64.h>
#include <linux/cpu.h>
unsigned int interactivity_req;
unsigned int expected_interval;
unsigned long nr_iowaiters, cpu_load;
- int resume_latency = dev_pm_qos_read_value(device);
+ int resume_latency = dev_pm_qos_raw_read_value(device);
if (data->needs_update) {
menu_update(drv, dev);
config CRYPTO_DEV_ATMEL_TDES
tristate "Support for Atmel DES/TDES hw accelerator"
+ depends on HAS_DMA
depends on ARCH_AT91 || COMPILE_TEST
select CRYPTO_DES
select CRYPTO_BLKCIPHER
config CRYPTO_DEV_ATMEL_SHA
tristate "Support for Atmel SHA hw accelerator"
+ depends on HAS_DMA
depends on ARCH_AT91 || COMPILE_TEST
select CRYPTO_HASH
help
config CRYPTO_DEV_MEDIATEK
tristate "MediaTek's EIP97 Cryptographic Engine driver"
+ depends on HAS_DMA
depends on (ARM && ARCH_MEDIATEK) || COMPILE_TEST
select CRYPTO_AES
select CRYPTO_AEAD
if (!curr->head) {
dev_err(&pdev->dev, "Command Q (%d) chunk (%d) allocation failed\n",
i, queue->nchunks);
+ kfree(curr);
goto cmd_qfail;
}
{
struct cpt_vf *cptvf = pci_get_drvdata(pdev);
- if (!cptvf)
+ if (!cptvf) {
dev_err(&pdev->dev, "Invalid CPT-VF device\n");
+ return;
+ }
/* Convey DOWN to PF */
if (cptvf_send_vf_down(cptvf)) {
{
struct pci_dev *pdev = cptvf->pdev;
- if (!info || !cptvf) {
- dev_err(&pdev->dev, "Input params are incorrect for post processing\n");
+ if (!info) {
+ dev_err(&pdev->dev, "incorrect cpt_info_buffer for post processing\n");
return;
}
void cryp_flush_inoutfifo(struct cryp_device_data *device_data)
{
/*
- * We always need to disble the hardware before trying to flush the
+ * We always need to disable the hardware before trying to flush the
* FIFO. This is something that isn't written in the design
* specification, but we have been informed by the hardware designers
* that this must be done.
}
ret = vi->vdev->config->find_vqs(vi->vdev, total_vqs, vqs, callbacks,
- names);
+ names, NULL);
if (ret)
goto err_find;
#include <asm/switch_to.h>
#include <crypto/aes.h>
#include <crypto/scatterwalk.h>
+#include <crypto/skcipher.h>
#include "aesp8-ppc.h"
struct p8_aes_cbc_ctx {
- struct crypto_blkcipher *fallback;
+ struct crypto_skcipher *fallback;
struct aes_key enc_key;
struct aes_key dec_key;
};
static int p8_aes_cbc_init(struct crypto_tfm *tfm)
{
const char *alg;
- struct crypto_blkcipher *fallback;
+ struct crypto_skcipher *fallback;
struct p8_aes_cbc_ctx *ctx = crypto_tfm_ctx(tfm);
if (!(alg = crypto_tfm_alg_name(tfm))) {
return -ENOENT;
}
- fallback =
- crypto_alloc_blkcipher(alg, 0, CRYPTO_ALG_NEED_FALLBACK);
+ fallback = crypto_alloc_skcipher(alg, 0,
+ CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK);
+
if (IS_ERR(fallback)) {
printk(KERN_ERR
"Failed to allocate transformation for '%s': %ld\n",
return PTR_ERR(fallback);
}
printk(KERN_INFO "Using '%s' as fallback implementation.\n",
- crypto_tfm_alg_driver_name((struct crypto_tfm *) fallback));
+ crypto_skcipher_driver_name(fallback));
+
- crypto_blkcipher_set_flags(
+ crypto_skcipher_set_flags(
fallback,
- crypto_blkcipher_get_flags((struct crypto_blkcipher *)tfm));
+ crypto_skcipher_get_flags((struct crypto_skcipher *)tfm));
ctx->fallback = fallback;
return 0;
struct p8_aes_cbc_ctx *ctx = crypto_tfm_ctx(tfm);
if (ctx->fallback) {
- crypto_free_blkcipher(ctx->fallback);
+ crypto_free_skcipher(ctx->fallback);
ctx->fallback = NULL;
}
}
pagefault_enable();
preempt_enable();
- ret += crypto_blkcipher_setkey(ctx->fallback, key, keylen);
+ ret += crypto_skcipher_setkey(ctx->fallback, key, keylen);
return ret;
}
struct blkcipher_walk walk;
struct p8_aes_cbc_ctx *ctx =
crypto_tfm_ctx(crypto_blkcipher_tfm(desc->tfm));
- struct blkcipher_desc fallback_desc = {
- .tfm = ctx->fallback,
- .info = desc->info,
- .flags = desc->flags
- };
if (in_interrupt()) {
- ret = crypto_blkcipher_encrypt(&fallback_desc, dst, src,
- nbytes);
+ SKCIPHER_REQUEST_ON_STACK(req, ctx->fallback);
+ skcipher_request_set_tfm(req, ctx->fallback);
+ skcipher_request_set_callback(req, desc->flags, NULL, NULL);
+ skcipher_request_set_crypt(req, src, dst, nbytes, desc->info);
+ ret = crypto_skcipher_encrypt(req);
+ skcipher_request_zero(req);
} else {
preempt_disable();
pagefault_disable();
struct blkcipher_walk walk;
struct p8_aes_cbc_ctx *ctx =
crypto_tfm_ctx(crypto_blkcipher_tfm(desc->tfm));
- struct blkcipher_desc fallback_desc = {
- .tfm = ctx->fallback,
- .info = desc->info,
- .flags = desc->flags
- };
if (in_interrupt()) {
- ret = crypto_blkcipher_decrypt(&fallback_desc, dst, src,
- nbytes);
+ SKCIPHER_REQUEST_ON_STACK(req, ctx->fallback);
+ skcipher_request_set_tfm(req, ctx->fallback);
+ skcipher_request_set_callback(req, desc->flags, NULL, NULL);
+ skcipher_request_set_crypt(req, src, dst, nbytes, desc->info);
+ ret = crypto_skcipher_decrypt(req);
+ skcipher_request_zero(req);
} else {
preempt_disable();
pagefault_disable();
#include <crypto/aes.h>
#include <crypto/scatterwalk.h>
#include <crypto/xts.h>
+#include <crypto/skcipher.h>
#include "aesp8-ppc.h"
struct p8_aes_xts_ctx {
- struct crypto_blkcipher *fallback;
+ struct crypto_skcipher *fallback;
struct aes_key enc_key;
struct aes_key dec_key;
struct aes_key tweak_key;
static int p8_aes_xts_init(struct crypto_tfm *tfm)
{
const char *alg;
- struct crypto_blkcipher *fallback;
+ struct crypto_skcipher *fallback;
struct p8_aes_xts_ctx *ctx = crypto_tfm_ctx(tfm);
if (!(alg = crypto_tfm_alg_name(tfm))) {
return -ENOENT;
}
- fallback =
- crypto_alloc_blkcipher(alg, 0, CRYPTO_ALG_NEED_FALLBACK);
+ fallback = crypto_alloc_skcipher(alg, 0,
+ CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK);
if (IS_ERR(fallback)) {
printk(KERN_ERR
"Failed to allocate transformation for '%s': %ld\n",
return PTR_ERR(fallback);
}
printk(KERN_INFO "Using '%s' as fallback implementation.\n",
- crypto_tfm_alg_driver_name((struct crypto_tfm *) fallback));
+ crypto_skcipher_driver_name(fallback));
- crypto_blkcipher_set_flags(
+ crypto_skcipher_set_flags(
fallback,
- crypto_blkcipher_get_flags((struct crypto_blkcipher *)tfm));
+ crypto_skcipher_get_flags((struct crypto_skcipher *)tfm));
ctx->fallback = fallback;
return 0;
struct p8_aes_xts_ctx *ctx = crypto_tfm_ctx(tfm);
if (ctx->fallback) {
- crypto_free_blkcipher(ctx->fallback);
+ crypto_free_skcipher(ctx->fallback);
ctx->fallback = NULL;
}
}
pagefault_enable();
preempt_enable();
- ret += crypto_blkcipher_setkey(ctx->fallback, key, keylen);
+ ret += crypto_skcipher_setkey(ctx->fallback, key, keylen);
return ret;
}
struct blkcipher_walk walk;
struct p8_aes_xts_ctx *ctx =
crypto_tfm_ctx(crypto_blkcipher_tfm(desc->tfm));
- struct blkcipher_desc fallback_desc = {
- .tfm = ctx->fallback,
- .info = desc->info,
- .flags = desc->flags
- };
if (in_interrupt()) {
- ret = enc ? crypto_blkcipher_encrypt(&fallback_desc, dst, src, nbytes) :
- crypto_blkcipher_decrypt(&fallback_desc, dst, src, nbytes);
+ SKCIPHER_REQUEST_ON_STACK(req, ctx->fallback);
+ skcipher_request_set_tfm(req, ctx->fallback);
+ skcipher_request_set_callback(req, desc->flags, NULL, NULL);
+ skcipher_request_set_crypt(req, src, dst, nbytes, desc->info);
+ ret = enc? crypto_skcipher_encrypt(req) : crypto_skcipher_decrypt(req);
+ skcipher_request_zero(req);
} else {
preempt_disable();
pagefault_disable();
#include <linux/pagemap.h>
#include <linux/module.h>
#include <linux/device.h>
+#include <linux/magic.h>
#include <linux/mount.h>
#include <linux/pfn_t.h>
#include <linux/hash.h>
#include <linux/export.h>
#include <linux/atomic.h>
#include <linux/dma-fence.h>
+#include <linux/sched/signal.h>
#define CREATE_TRACE_POINTS
#include <trace/events/dma_fence.h>
#include <linux/freezer.h>
#include <linux/init.h>
#include <linux/kthread.h>
+#include <linux/sched/task.h>
#include <linux/module.h>
#include <linux/moduleparam.h>
#include <linux/random.h>
return 0;
}
}
- pr_err_once("requested map not found.\n");
return -ENOENT;
}
rc = efi_mem_desc_lookup(efi.esrt, &md);
if (rc < 0) {
- pr_err("ESRT header is not in the memory map.\n");
+ pr_warn("ESRT header is not in the memory map.\n");
return;
}
#include <linux/cpu_pm.h>
#include <linux/kernel.h>
#include <linux/kthread.h>
+#include <uapi/linux/sched/types.h>
#include <linux/module.h>
#include <linux/preempt.h>
#include <linux/psci.h>
#include <linux/of_device.h>
#include <linux/platform_device.h>
#include <linux/semaphore.h>
+#include <linux/sched/clock.h>
#include <soc/tegra/bpmp.h>
#include <soc/tegra/bpmp-abi.h>
}
r = amdgpu_late_init(adev);
- if (r)
+ if (r) {
+ if (fbcon)
+ console_unlock();
return r;
+ }
/* pin cursors */
list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) {
* - 3.7.0 - Add support for VCE clock list packet
* - 3.8.0 - Add support raster config init in the kernel
* - 3.9.0 - Add support for memory query info about VRAM and GTT.
+ * - 3.10.0 - Add support for new fences ioctl, new gem ioctl flags
*/
#define KMS_DRIVER_MAJOR 3
-#define KMS_DRIVER_MINOR 9
+#define KMS_DRIVER_MINOR 10
#define KMS_DRIVER_PATCHLEVEL 0
int amdgpu_vram_limit = 0;
bool kernel = false;
int r;
+ /* reject invalid gem flags */
+ if (args->in.domain_flags & ~(AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED |
+ AMDGPU_GEM_CREATE_NO_CPU_ACCESS |
+ AMDGPU_GEM_CREATE_CPU_GTT_USWC |
+ AMDGPU_GEM_CREATE_VRAM_CLEARED|
+ AMDGPU_GEM_CREATE_SHADOW |
+ AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS)) {
+ r = -EINVAL;
+ goto error_unlock;
+ }
+ /* reject invalid gem domains */
+ if (args->in.domains & ~(AMDGPU_GEM_DOMAIN_CPU |
+ AMDGPU_GEM_DOMAIN_GTT |
+ AMDGPU_GEM_DOMAIN_VRAM |
+ AMDGPU_GEM_DOMAIN_GDS |
+ AMDGPU_GEM_DOMAIN_GWS |
+ AMDGPU_GEM_DOMAIN_OA)) {
+ r = -EINVAL;
+ goto error_unlock;
+ }
+
/* create a gem object to contain this object in */
if (args->in.domains & (AMDGPU_GEM_DOMAIN_GDS |
AMDGPU_GEM_DOMAIN_GWS | AMDGPU_GEM_DOMAIN_OA)) {
}
}
- /* disble sdma engine before programing it */
+ /* disable sdma engine before programing it */
sdma_v3_0_ctx_switch_enable(adev, false);
sdma_v3_0_enable(adev, false);
#include <linux/mm_types.h>
#include <linux/slab.h>
#include <linux/types.h>
-#include <linux/sched.h>
+#include <linux/sched/signal.h>
#include <linux/uaccess.h>
#include <linux/mm.h>
#include <linux/mman.h>
#include <linux/printk.h>
#include <linux/slab.h>
+#include <linux/mm_types.h>
+
#include "kfd_priv.h"
#include "kfd_mqd_manager.h"
#include "cik_regs.h"
#include <linux/printk.h>
#include <linux/slab.h>
+#include <linux/mm_types.h>
+
#include "kfd_priv.h"
#include "kfd_mqd_manager.h"
#include "vi_structs.h"
#include <linux/mutex.h>
#include <linux/log2.h>
#include <linux/sched.h>
+#include <linux/sched/mm.h>
#include <linux/slab.h>
#include <linux/amd-iommu.h>
#include <linux/notifier.h>
#include <linux/kthread.h>
#include <linux/wait.h>
#include <linux/sched.h>
+#include <uapi/linux/sched/types.h>
#include <drm/drmP.h>
#include "gpu_scheduler.h"
/* Panel in Samsung NP700G7A-S01PL notebook reports 6bpc */
{ "SEC", 0xd033, EDID_QUIRK_FORCE_8BPC },
+
+ /* Rotel RSX-1058 forwards sink's EDID but only does HDMI 1.1*/
+ { "ETR", 13896, EDID_QUIRK_FORCE_8BPC },
};
/*
*/
#include <linux/export.h>
+#include <linux/sched/signal.h>
+
#include <drm/drmP.h>
#include "drm_legacy.h"
#include "drm_internal.h"
#include <linux/spinlock.h>
#include <linux/shmem_fs.h>
+#include <linux/sched/mm.h>
+#include <linux/sched/task.h>
#include "etnaviv_drv.h"
#include "etnaviv_gem.h"
INTEL_GVT_PCI_BAR_MAX,
};
+/* bitmap for writable bits (RW or RW1C bits, but cannot co-exist in one
+ * byte) byte by byte in standard pci configuration space. (not the full
+ * 256 bytes.)
+ */
+static const u8 pci_cfg_space_rw_bmp[PCI_INTERRUPT_LINE + 4] = {
+ [PCI_COMMAND] = 0xff, 0x07,
+ [PCI_STATUS] = 0x00, 0xf9, /* the only one RW1C byte */
+ [PCI_CACHE_LINE_SIZE] = 0xff,
+ [PCI_BASE_ADDRESS_0 ... PCI_CARDBUS_CIS - 1] = 0xff,
+ [PCI_ROM_ADDRESS] = 0x01, 0xf8, 0xff, 0xff,
+ [PCI_INTERRUPT_LINE] = 0xff,
+};
+
+/**
+ * vgpu_pci_cfg_mem_write - write virtual cfg space memory
+ *
+ * Use this function to write virtual cfg space memory.
+ * For standard cfg space, only RW bits can be changed,
+ * and we emulates the RW1C behavior of PCI_STATUS register.
+ */
+static void vgpu_pci_cfg_mem_write(struct intel_vgpu *vgpu, unsigned int off,
+ u8 *src, unsigned int bytes)
+{
+ u8 *cfg_base = vgpu_cfg_space(vgpu);
+ u8 mask, new, old;
+ int i = 0;
+
+ for (; i < bytes && (off + i < sizeof(pci_cfg_space_rw_bmp)); i++) {
+ mask = pci_cfg_space_rw_bmp[off + i];
+ old = cfg_base[off + i];
+ new = src[i] & mask;
+
+ /**
+ * The PCI_STATUS high byte has RW1C bits, here
+ * emulates clear by writing 1 for these bits.
+ * Writing a 0b to RW1C bits has no effect.
+ */
+ if (off + i == PCI_STATUS + 1)
+ new = (~new & old) & mask;
+
+ cfg_base[off + i] = (old & ~mask) | new;
+ }
+
+ /* For other configuration space directly copy as it is. */
+ if (i < bytes)
+ memcpy(cfg_base + off + i, src + i, bytes - i);
+}
+
/**
* intel_vgpu_emulate_cfg_read - emulate vGPU configuration space read
*
u8 changed = old ^ new;
int ret;
- memcpy(vgpu_cfg_space(vgpu) + offset, p_data, bytes);
+ vgpu_pci_cfg_mem_write(vgpu, offset, p_data, bytes);
if (!(changed & PCI_COMMAND_MEMORY))
return 0;
{
int ret;
+ if (vgpu->failsafe)
+ return 0;
+
if (WARN_ON(bytes > 4))
return -EINVAL;
if (ret)
return ret;
- memcpy(vgpu_cfg_space(vgpu) + offset, p_data, bytes);
+ vgpu_pci_cfg_mem_write(vgpu, offset, p_data, bytes);
break;
default:
- memcpy(vgpu_cfg_space(vgpu) + offset, p_data, bytes);
+ vgpu_pci_cfg_mem_write(vgpu, offset, p_data, bytes);
break;
}
return 0;
if (d_info == NULL)
return;
- gvt_err("opcode=0x%x %s sub_ops:",
+ gvt_dbg_cmd("opcode=0x%x %s sub_ops:",
cmd >> (32 - d_info->op_len), d_info->name);
for (i = 0; i < d_info->nr_sub_op; i++)
int cnt = 0;
int i;
- gvt_err(" vgpu%d RING%d: ring_start(%08lx) ring_end(%08lx)"
+ gvt_dbg_cmd(" vgpu%d RING%d: ring_start(%08lx) ring_end(%08lx)"
" ring_head(%08lx) ring_tail(%08lx)\n", s->vgpu->id,
s->ring_id, s->ring_start, s->ring_start + s->ring_size,
s->ring_head, s->ring_tail);
- gvt_err(" %s %s ip_gma(%08lx) ",
+ gvt_dbg_cmd(" %s %s ip_gma(%08lx) ",
s->buf_type == RING_BUFFER_INSTRUCTION ?
"RING_BUFFER" : "BATCH_BUFFER",
s->buf_addr_type == GTT_BUFFER ?
"GTT" : "PPGTT", s->ip_gma);
if (s->ip_va == NULL) {
- gvt_err(" ip_va(NULL)");
+ gvt_dbg_cmd(" ip_va(NULL)");
return;
}
- gvt_err(" ip_va=%p: %08x %08x %08x %08x\n",
+ gvt_dbg_cmd(" ip_va=%p: %08x %08x %08x %08x\n",
s->ip_va, cmd_val(s, 0), cmd_val(s, 1),
cmd_val(s, 2), cmd_val(s, 3));
return 0;
}
+static unsigned char virtual_dp_monitor_edid[GVT_EDID_NUM][EDID_SIZE] = {
+ {
+/* EDID with 1024x768 as its resolution */
+ /*Header*/
+ 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x00,
+ /* Vendor & Product Identification */
+ 0x22, 0xf0, 0x54, 0x29, 0x00, 0x00, 0x00, 0x00, 0x04, 0x17,
+ /* Version & Revision */
+ 0x01, 0x04,
+ /* Basic Display Parameters & Features */
+ 0xa5, 0x34, 0x20, 0x78, 0x23,
+ /* Color Characteristics */
+ 0xfc, 0x81, 0xa4, 0x55, 0x4d, 0x9d, 0x25, 0x12, 0x50, 0x54,
+ /* Established Timings: maximum resolution is 1024x768 */
+ 0x21, 0x08, 0x00,
+ /* Standard Timings. All invalid */
+ 0x00, 0xc0, 0x00, 0xc0, 0x00, 0x40, 0x00, 0x80, 0x00, 0x00,
+ 0x00, 0x40, 0x00, 0x00, 0x00, 0x01,
+ /* 18 Byte Data Blocks 1: invalid */
+ 0x00, 0x00, 0x80, 0xa0, 0x70, 0xb0,
+ 0x23, 0x40, 0x30, 0x20, 0x36, 0x00, 0x06, 0x44, 0x21, 0x00, 0x00, 0x1a,
+ /* 18 Byte Data Blocks 2: invalid */
+ 0x00, 0x00, 0x00, 0xfd, 0x00, 0x18, 0x3c, 0x18, 0x50, 0x11, 0x00, 0x0a,
+ 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
+ /* 18 Byte Data Blocks 3: invalid */
+ 0x00, 0x00, 0x00, 0xfc, 0x00, 0x48,
+ 0x50, 0x20, 0x5a, 0x52, 0x32, 0x34, 0x34, 0x30, 0x77, 0x0a, 0x20, 0x20,
+ /* 18 Byte Data Blocks 4: invalid */
+ 0x00, 0x00, 0x00, 0xff, 0x00, 0x43, 0x4e, 0x34, 0x33, 0x30, 0x34, 0x30,
+ 0x44, 0x58, 0x51, 0x0a, 0x20, 0x20,
+ /* Extension Block Count */
+ 0x00,
+ /* Checksum */
+ 0xef,
+ },
+ {
/* EDID with 1920x1200 as its resolution */
-static unsigned char virtual_dp_monitor_edid[] = {
- /*Header*/
- 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x00,
- /* Vendor & Product Identification */
- 0x22, 0xf0, 0x54, 0x29, 0x00, 0x00, 0x00, 0x00, 0x04, 0x17,
- /* Version & Revision */
- 0x01, 0x04,
- /* Basic Display Parameters & Features */
- 0xa5, 0x34, 0x20, 0x78, 0x23,
- /* Color Characteristics */
- 0xfc, 0x81, 0xa4, 0x55, 0x4d, 0x9d, 0x25, 0x12, 0x50, 0x54,
- /* Established Timings: maximum resolution is 1024x768 */
- 0x21, 0x08, 0x00,
- /*
- * Standard Timings.
- * below new resolutions can be supported:
- * 1920x1080, 1280x720, 1280x960, 1280x1024,
- * 1440x900, 1600x1200, 1680x1050
- */
- 0xd1, 0xc0, 0x81, 0xc0, 0x81, 0x40, 0x81, 0x80, 0x95, 0x00,
- 0xa9, 0x40, 0xb3, 0x00, 0x01, 0x01,
- /* 18 Byte Data Blocks 1: max resolution is 1920x1200 */
- 0x28, 0x3c, 0x80, 0xa0, 0x70, 0xb0,
- 0x23, 0x40, 0x30, 0x20, 0x36, 0x00, 0x06, 0x44, 0x21, 0x00, 0x00, 0x1a,
- /* 18 Byte Data Blocks 2: invalid */
- 0x00, 0x00, 0x00, 0xfd, 0x00, 0x18, 0x3c, 0x18, 0x50, 0x11, 0x00, 0x0a,
- 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
- /* 18 Byte Data Blocks 3: invalid */
- 0x00, 0x00, 0x00, 0xfc, 0x00, 0x48,
- 0x50, 0x20, 0x5a, 0x52, 0x32, 0x34, 0x34, 0x30, 0x77, 0x0a, 0x20, 0x20,
- /* 18 Byte Data Blocks 4: invalid */
- 0x00, 0x00, 0x00, 0xff, 0x00, 0x43, 0x4e, 0x34, 0x33, 0x30, 0x34, 0x30,
- 0x44, 0x58, 0x51, 0x0a, 0x20, 0x20,
- /* Extension Block Count */
- 0x00,
- /* Checksum */
- 0x45,
+ /*Header*/
+ 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x00,
+ /* Vendor & Product Identification */
+ 0x22, 0xf0, 0x54, 0x29, 0x00, 0x00, 0x00, 0x00, 0x04, 0x17,
+ /* Version & Revision */
+ 0x01, 0x04,
+ /* Basic Display Parameters & Features */
+ 0xa5, 0x34, 0x20, 0x78, 0x23,
+ /* Color Characteristics */
+ 0xfc, 0x81, 0xa4, 0x55, 0x4d, 0x9d, 0x25, 0x12, 0x50, 0x54,
+ /* Established Timings: maximum resolution is 1024x768 */
+ 0x21, 0x08, 0x00,
+ /*
+ * Standard Timings.
+ * below new resolutions can be supported:
+ * 1920x1080, 1280x720, 1280x960, 1280x1024,
+ * 1440x900, 1600x1200, 1680x1050
+ */
+ 0xd1, 0xc0, 0x81, 0xc0, 0x81, 0x40, 0x81, 0x80, 0x95, 0x00,
+ 0xa9, 0x40, 0xb3, 0x00, 0x01, 0x01,
+ /* 18 Byte Data Blocks 1: max resolution is 1920x1200 */
+ 0x28, 0x3c, 0x80, 0xa0, 0x70, 0xb0,
+ 0x23, 0x40, 0x30, 0x20, 0x36, 0x00, 0x06, 0x44, 0x21, 0x00, 0x00, 0x1a,
+ /* 18 Byte Data Blocks 2: invalid */
+ 0x00, 0x00, 0x00, 0xfd, 0x00, 0x18, 0x3c, 0x18, 0x50, 0x11, 0x00, 0x0a,
+ 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
+ /* 18 Byte Data Blocks 3: invalid */
+ 0x00, 0x00, 0x00, 0xfc, 0x00, 0x48,
+ 0x50, 0x20, 0x5a, 0x52, 0x32, 0x34, 0x34, 0x30, 0x77, 0x0a, 0x20, 0x20,
+ /* 18 Byte Data Blocks 4: invalid */
+ 0x00, 0x00, 0x00, 0xff, 0x00, 0x43, 0x4e, 0x34, 0x33, 0x30, 0x34, 0x30,
+ 0x44, 0x58, 0x51, 0x0a, 0x20, 0x20,
+ /* Extension Block Count */
+ 0x00,
+ /* Checksum */
+ 0x45,
+ },
};
#define DPCD_HEADER_SIZE 0xb
vgpu_vreg(vgpu, SDEISR) &= ~(SDE_PORTA_HOTPLUG_SPT |
SDE_PORTE_HOTPLUG_SPT);
- if (intel_vgpu_has_monitor_on_port(vgpu, PORT_B))
+ if (intel_vgpu_has_monitor_on_port(vgpu, PORT_B)) {
vgpu_vreg(vgpu, SDEISR) |= SDE_PORTB_HOTPLUG_CPT;
+ vgpu_vreg(vgpu, SFUSE_STRAP) |= SFUSE_STRAP_DDIB_DETECTED;
+ }
- if (intel_vgpu_has_monitor_on_port(vgpu, PORT_C))
+ if (intel_vgpu_has_monitor_on_port(vgpu, PORT_C)) {
vgpu_vreg(vgpu, SDEISR) |= SDE_PORTC_HOTPLUG_CPT;
+ vgpu_vreg(vgpu, SFUSE_STRAP) |= SFUSE_STRAP_DDIC_DETECTED;
+ }
- if (intel_vgpu_has_monitor_on_port(vgpu, PORT_D))
+ if (intel_vgpu_has_monitor_on_port(vgpu, PORT_D)) {
vgpu_vreg(vgpu, SDEISR) |= SDE_PORTD_HOTPLUG_CPT;
+ vgpu_vreg(vgpu, SFUSE_STRAP) |= SFUSE_STRAP_DDID_DETECTED;
+ }
if (IS_SKYLAKE(dev_priv) &&
intel_vgpu_has_monitor_on_port(vgpu, PORT_E)) {
GEN8_PORT_DP_A_HOTPLUG;
else
vgpu_vreg(vgpu, SDEISR) |= SDE_PORTA_HOTPLUG_SPT;
+
+ vgpu_vreg(vgpu, DDI_BUF_CTL(PORT_A)) |= DDI_INIT_DISPLAY_DETECTED;
}
}
}
static int setup_virtual_dp_monitor(struct intel_vgpu *vgpu, int port_num,
- int type)
+ int type, unsigned int resolution)
{
struct intel_vgpu_port *port = intel_vgpu_port(vgpu, port_num);
+ if (WARN_ON(resolution >= GVT_EDID_NUM))
+ return -EINVAL;
+
port->edid = kzalloc(sizeof(*(port->edid)), GFP_KERNEL);
if (!port->edid)
return -ENOMEM;
return -ENOMEM;
}
- memcpy(port->edid->edid_block, virtual_dp_monitor_edid,
+ memcpy(port->edid->edid_block, virtual_dp_monitor_edid[resolution],
EDID_SIZE);
port->edid->data_valid = true;
* Zero on success, negative error code if failed.
*
*/
-int intel_vgpu_init_display(struct intel_vgpu *vgpu)
+int intel_vgpu_init_display(struct intel_vgpu *vgpu, u64 resolution)
{
struct drm_i915_private *dev_priv = vgpu->gvt->dev_priv;
intel_vgpu_init_i2c_edid(vgpu);
if (IS_SKYLAKE(dev_priv))
- return setup_virtual_dp_monitor(vgpu, PORT_D, GVT_DP_D);
+ return setup_virtual_dp_monitor(vgpu, PORT_D, GVT_DP_D,
+ resolution);
else
- return setup_virtual_dp_monitor(vgpu, PORT_B, GVT_DP_B);
+ return setup_virtual_dp_monitor(vgpu, PORT_B, GVT_DP_B,
+ resolution);
}
/**
int type;
};
+enum intel_vgpu_edid {
+ GVT_EDID_1024_768,
+ GVT_EDID_1920_1200,
+ GVT_EDID_NUM,
+};
+
+static inline char *vgpu_edid_str(enum intel_vgpu_edid id)
+{
+ switch (id) {
+ case GVT_EDID_1024_768:
+ return "1024x768";
+ case GVT_EDID_1920_1200:
+ return "1920x1200";
+ default:
+ return "";
+ }
+}
+
void intel_gvt_emulate_vblank(struct intel_gvt *gvt);
void intel_gvt_check_vblank_emulation(struct intel_gvt *gvt);
-int intel_vgpu_init_display(struct intel_vgpu *vgpu);
+int intel_vgpu_init_display(struct intel_vgpu *vgpu, u64 resolution);
void intel_vgpu_reset_display(struct intel_vgpu *vgpu);
void intel_vgpu_clean_display(struct intel_vgpu *vgpu);
int ret;
size = sizeof(*h) + info->mmio_size + info->cfg_space_size - 1;
- firmware = vmalloc(size);
+ firmware = vzalloc(size);
if (!firmware)
return -ENOMEM;
gma = g_gtt_index << GTT_PAGE_SHIFT;
/* the VM may configure the whole GM space when ballooning is used */
- if (WARN_ONCE(!vgpu_gmadr_is_valid(vgpu, gma),
- "vgpu%d: found oob ggtt write, offset %x\n",
- vgpu->id, off)) {
+ if (!vgpu_gmadr_is_valid(vgpu, gma))
return 0;
- }
ggtt_get_guest_entry(ggtt_mm, &e, g_gtt_index);
return create_scratch_page_tree(vgpu);
}
+static void intel_vgpu_free_mm(struct intel_vgpu *vgpu, int type)
+{
+ struct list_head *pos, *n;
+ struct intel_vgpu_mm *mm;
+
+ list_for_each_safe(pos, n, &vgpu->gtt.mm_list_head) {
+ mm = container_of(pos, struct intel_vgpu_mm, list);
+ if (mm->type == type) {
+ vgpu->gvt->gtt.mm_free_page_table(mm);
+ list_del(&mm->list);
+ list_del(&mm->lru_list);
+ kfree(mm);
+ }
+ }
+}
+
/**
* intel_vgpu_clean_gtt - clean up per-vGPU graphics memory virulization
* @vgpu: a vGPU
*/
void intel_vgpu_clean_gtt(struct intel_vgpu *vgpu)
{
- struct list_head *pos, *n;
- struct intel_vgpu_mm *mm;
-
ppgtt_free_all_shadow_page(vgpu);
release_scratch_page_tree(vgpu);
- list_for_each_safe(pos, n, &vgpu->gtt.mm_list_head) {
- mm = container_of(pos, struct intel_vgpu_mm, list);
- vgpu->gvt->gtt.mm_free_page_table(mm);
- list_del(&mm->list);
- list_del(&mm->lru_list);
- kfree(mm);
- }
+ intel_vgpu_free_mm(vgpu, INTEL_GVT_MM_PPGTT);
+ intel_vgpu_free_mm(vgpu, INTEL_GVT_MM_GGTT);
}
static void clean_spt_oos(struct intel_gvt *gvt)
int i;
ppgtt_free_all_shadow_page(vgpu);
+
+ /* Shadow pages are only created when there is no page
+ * table tracking data, so remove page tracking data after
+ * removing the shadow pages.
+ */
+ intel_vgpu_free_mm(vgpu, INTEL_GVT_MM_PPGTT);
+
if (!dmlr)
return;
int id;
unsigned long handle; /* vGPU handle used by hypervisor MPT modules */
bool active;
+ bool pv_notified;
+ bool failsafe;
bool resetting;
void *sched_data;
};
struct intel_gvt_opregion {
- void __iomem *opregion_va;
+ void *opregion_va;
u32 opregion_pa;
};
#define NR_MAX_INTEL_VGPU_TYPES 20
struct intel_vgpu_type {
char name[16];
- unsigned int max_instance;
unsigned int avail_instance;
unsigned int low_gm_size;
unsigned int high_gm_size;
unsigned int fence;
+ enum intel_vgpu_edid resolution;
};
struct intel_gvt {
__u64 low_gm_sz; /* in MB */
__u64 high_gm_sz; /* in MB */
__u64 fence_sz;
+ __u64 resolution;
__s32 primary;
__u64 vgpu_id;
};
};
+enum {
+ GVT_FAILSAFE_UNSUPPORTED_GUEST,
+ GVT_FAILSAFE_INSUFFICIENT_RESOURCE,
+};
+
#include "mpt.h"
#endif
info->size = size;
info->length = (i + 4) < end ? 4 : (end - i);
info->addr_mask = addr_mask;
+ info->ro_mask = ro_mask;
info->device = device;
info->read = read ? read : intel_vgpu_default_mmio_read;
info->write = write ? write : intel_vgpu_default_mmio_write;
#define fence_num_to_offset(num) \
(num * 8 + i915_mmio_reg_offset(FENCE_REG_GEN6_LO(0)))
+
+static void enter_failsafe_mode(struct intel_vgpu *vgpu, int reason)
+{
+ switch (reason) {
+ case GVT_FAILSAFE_UNSUPPORTED_GUEST:
+ pr_err("Detected your guest driver doesn't support GVT-g.\n");
+ break;
+ case GVT_FAILSAFE_INSUFFICIENT_RESOURCE:
+ pr_err("Graphics resource is not enough for the guest\n");
+ default:
+ break;
+ }
+ pr_err("Now vgpu %d will enter failsafe mode.\n", vgpu->id);
+ vgpu->failsafe = true;
+}
+
static int sanitize_fence_mmio_access(struct intel_vgpu *vgpu,
unsigned int fence_num, void *p_data, unsigned int bytes)
{
if (fence_num >= vgpu_fence_sz(vgpu)) {
- gvt_err("vgpu%d: found oob fence register access\n",
- vgpu->id);
- gvt_err("vgpu%d: total fence num %d access fence num %d\n",
- vgpu->id, vgpu_fence_sz(vgpu), fence_num);
+
+ /* When guest access oob fence regs without access
+ * pv_info first, we treat guest not supporting GVT,
+ * and we will let vgpu enter failsafe mode.
+ */
+ if (!vgpu->pv_notified)
+ enter_failsafe_mode(vgpu,
+ GVT_FAILSAFE_UNSUPPORTED_GUEST);
+
+ if (!vgpu->mmio.disable_warn_untrack) {
+ gvt_err("vgpu%d: found oob fence register access\n",
+ vgpu->id);
+ gvt_err("vgpu%d: total fence %d, access fence %d\n",
+ vgpu->id, vgpu_fence_sz(vgpu),
+ fence_num);
+ }
memset(p_data, 0, bytes);
+ return -EINVAL;
}
return 0;
}
return 0;
}
+/* ascendingly sorted */
+static i915_reg_t force_nonpriv_white_list[] = {
+ GEN9_CS_DEBUG_MODE1, //_MMIO(0x20ec)
+ GEN9_CTX_PREEMPT_REG,//_MMIO(0x2248)
+ GEN8_CS_CHICKEN1,//_MMIO(0x2580)
+ _MMIO(0x2690),
+ _MMIO(0x2694),
+ _MMIO(0x2698),
+ _MMIO(0x4de0),
+ _MMIO(0x4de4),
+ _MMIO(0x4dfc),
+ GEN7_COMMON_SLICE_CHICKEN1,//_MMIO(0x7010)
+ _MMIO(0x7014),
+ HDC_CHICKEN0,//_MMIO(0x7300)
+ GEN8_HDC_CHICKEN1,//_MMIO(0x7304)
+ _MMIO(0x7700),
+ _MMIO(0x7704),
+ _MMIO(0x7708),
+ _MMIO(0x770c),
+ _MMIO(0xb110),
+ GEN8_L3SQCREG4,//_MMIO(0xb118)
+ _MMIO(0xe100),
+ _MMIO(0xe18c),
+ _MMIO(0xe48c),
+ _MMIO(0xe5f4),
+};
+
+/* a simple bsearch */
+static inline bool in_whitelist(unsigned int reg)
+{
+ int left = 0, right = ARRAY_SIZE(force_nonpriv_white_list);
+ i915_reg_t *array = force_nonpriv_white_list;
+
+ while (left < right) {
+ int mid = (left + right)/2;
+
+ if (reg > array[mid].reg)
+ left = mid + 1;
+ else if (reg < array[mid].reg)
+ right = mid;
+ else
+ return true;
+ }
+ return false;
+}
+
+static int force_nonpriv_write(struct intel_vgpu *vgpu,
+ unsigned int offset, void *p_data, unsigned int bytes)
+{
+ u32 reg_nonpriv = *(u32 *)p_data;
+ int ret = -EINVAL;
+
+ if ((bytes != 4) || ((offset & (bytes - 1)) != 0)) {
+ gvt_err("vgpu(%d) Invalid FORCE_NONPRIV offset %x(%dB)\n",
+ vgpu->id, offset, bytes);
+ return ret;
+ }
+
+ if (in_whitelist(reg_nonpriv)) {
+ ret = intel_vgpu_default_mmio_write(vgpu, offset, p_data,
+ bytes);
+ } else {
+ gvt_err("vgpu(%d) Invalid FORCE_NONPRIV write %x\n",
+ vgpu->id, reg_nonpriv);
+ }
+ return ret;
+}
+
static int ddi_buf_ctl_mmio_write(struct intel_vgpu *vgpu, unsigned int offset,
void *p_data, unsigned int bytes)
{
if (invalid_read)
gvt_err("invalid pvinfo read: [%x:%x] = %x\n",
offset, bytes, *(u32 *)p_data);
+ vgpu->pv_notified = true;
return 0;
}
char vmid_str[20];
char display_ready_str[20];
- snprintf(display_ready_str, 20, "GVT_DISPLAY_READY=%d\n", ready);
+ snprintf(display_ready_str, 20, "GVT_DISPLAY_READY=%d", ready);
env[0] = display_ready_str;
snprintf(vmid_str, 20, "VMID=%d", vgpu->id);
case _vgtif_reg(execlist_context_descriptor_lo):
case _vgtif_reg(execlist_context_descriptor_hi):
break;
+ case _vgtif_reg(rsv5[0])..._vgtif_reg(rsv5[3]):
+ enter_failsafe_mode(vgpu, GVT_FAILSAFE_INSUFFICIENT_RESOURCE);
+ break;
default:
gvt_err("invalid pvinfo write offset %x bytes %x data %x\n",
offset, bytes, data);
u32 *data0 = &vgpu_vreg(vgpu, GEN6_PCODE_DATA);
switch (cmd) {
- case 0x6:
- /**
- * "Read memory latency" command on gen9.
- * Below memory latency values are read
- * from skylake platform.
- */
- if (!*data0)
- *data0 = 0x1e1a1100;
- else
- *data0 = 0x61514b3d;
+ case GEN9_PCODE_READ_MEM_LATENCY:
+ if (IS_SKYLAKE(vgpu->gvt->dev_priv)) {
+ /**
+ * "Read memory latency" command on gen9.
+ * Below memory latency values are read
+ * from skylake platform.
+ */
+ if (!*data0)
+ *data0 = 0x1e1a1100;
+ else
+ *data0 = 0x61514b3d;
+ }
+ break;
+ case SKL_PCODE_CDCLK_CONTROL:
+ if (IS_SKYLAKE(vgpu->gvt->dev_priv))
+ *data0 = SKL_CDCLK_READY_FOR_CHANGE;
break;
- case 0x5:
+ case GEN6_PCODE_READ_RC6VIDS:
*data0 |= 0x1;
break;
}
gvt_dbg_core("VM(%d) write %x to mailbox, return data0 %x\n",
vgpu->id, value, *data0);
-
- value &= ~(1 << 31);
+ /**
+ * PCODE_READY clear means ready for pcode read/write,
+ * PCODE_ERROR_MASK clear means no error happened. In GVT-g we
+ * always emulate as pcode read/write success and ready for access
+ * anytime, since we don't touch real physical registers here.
+ */
+ value &= ~(GEN6_PCODE_READY | GEN6_PCODE_ERROR_MASK);
return intel_vgpu_default_mmio_write(vgpu, offset, &value, bytes);
}
bool enable_execlist;
write_vreg(vgpu, offset, p_data, bytes);
+
+ /* when PPGTT mode enabled, we will check if guest has called
+ * pvinfo, if not, we will treat this guest as non-gvtg-aware
+ * guest, and stop emulating its cfg space, mmio, gtt, etc.
+ */
+ if (((data & _MASKED_BIT_ENABLE(GFX_PPGTT_ENABLE)) ||
+ (data & _MASKED_BIT_ENABLE(GFX_RUN_LIST_ENABLE)))
+ && !vgpu->pv_notified) {
+ enter_failsafe_mode(vgpu, GVT_FAILSAFE_UNSUPPORTED_GUEST);
+ return 0;
+ }
if ((data & _MASKED_BIT_ENABLE(GFX_RUN_LIST_ENABLE))
|| (data & _MASKED_BIT_DISABLE(GFX_RUN_LIST_ENABLE))) {
enable_execlist = !!(data & GFX_RUN_LIST_ENABLE);
#define MMIO_GM(reg, d, r, w) \
MMIO_F(reg, 4, F_GMADR, 0xFFFFF000, 0, d, r, w)
+#define MMIO_GM_RDR(reg, d, r, w) \
+ MMIO_F(reg, 4, F_GMADR | F_CMD_ACCESS, 0xFFFFF000, 0, d, r, w)
+
#define MMIO_RO(reg, d, f, rm, r, w) \
MMIO_F(reg, 4, F_RO | f, 0, rm, d, r, w)
#define MMIO_RING_GM(prefix, d, r, w) \
MMIO_RING_F(prefix, 4, F_GMADR, 0xFFFF0000, 0, d, r, w)
+#define MMIO_RING_GM_RDR(prefix, d, r, w) \
+ MMIO_RING_F(prefix, 4, F_GMADR | F_CMD_ACCESS, 0xFFFF0000, 0, d, r, w)
+
#define MMIO_RING_RO(prefix, d, f, rm, r, w) \
MMIO_RING_F(prefix, 4, F_RO | f, 0, rm, d, r, w)
struct drm_i915_private *dev_priv = gvt->dev_priv;
int ret;
- MMIO_RING_DFH(RING_IMR, D_ALL, 0, NULL, intel_vgpu_reg_imr_handler);
+ MMIO_RING_DFH(RING_IMR, D_ALL, F_CMD_ACCESS, NULL,
+ intel_vgpu_reg_imr_handler);
MMIO_DFH(SDEIMR, D_ALL, 0, NULL, intel_vgpu_reg_imr_handler);
MMIO_DFH(SDEIER, D_ALL, 0, NULL, intel_vgpu_reg_ier_handler);
MMIO_DFH(SDEIIR, D_ALL, 0, NULL, intel_vgpu_reg_iir_handler);
MMIO_D(SDEISR, D_ALL);
- MMIO_RING_D(RING_HWSTAM, D_ALL);
+ MMIO_RING_DFH(RING_HWSTAM, D_ALL, F_CMD_ACCESS, NULL, NULL);
- MMIO_GM(RENDER_HWS_PGA_GEN7, D_ALL, NULL, NULL);
- MMIO_GM(BSD_HWS_PGA_GEN7, D_ALL, NULL, NULL);
- MMIO_GM(BLT_HWS_PGA_GEN7, D_ALL, NULL, NULL);
- MMIO_GM(VEBOX_HWS_PGA_GEN7, D_ALL, NULL, NULL);
+ MMIO_GM_RDR(RENDER_HWS_PGA_GEN7, D_ALL, NULL, NULL);
+ MMIO_GM_RDR(BSD_HWS_PGA_GEN7, D_ALL, NULL, NULL);
+ MMIO_GM_RDR(BLT_HWS_PGA_GEN7, D_ALL, NULL, NULL);
+ MMIO_GM_RDR(VEBOX_HWS_PGA_GEN7, D_ALL, NULL, NULL);
#define RING_REG(base) (base + 0x28)
- MMIO_RING_D(RING_REG, D_ALL);
+ MMIO_RING_DFH(RING_REG, D_ALL, F_CMD_ACCESS, NULL, NULL);
#undef RING_REG
#define RING_REG(base) (base + 0x134)
- MMIO_RING_D(RING_REG, D_ALL);
+ MMIO_RING_DFH(RING_REG, D_ALL, F_CMD_ACCESS, NULL, NULL);
#undef RING_REG
- MMIO_GM(0x2148, D_ALL, NULL, NULL);
- MMIO_GM(CCID, D_ALL, NULL, NULL);
- MMIO_GM(0x12198, D_ALL, NULL, NULL);
+ MMIO_GM_RDR(0x2148, D_ALL, NULL, NULL);
+ MMIO_GM_RDR(CCID, D_ALL, NULL, NULL);
+ MMIO_GM_RDR(0x12198, D_ALL, NULL, NULL);
MMIO_D(GEN7_CXT_SIZE, D_ALL);
- MMIO_RING_D(RING_TAIL, D_ALL);
- MMIO_RING_D(RING_HEAD, D_ALL);
- MMIO_RING_D(RING_CTL, D_ALL);
- MMIO_RING_D(RING_ACTHD, D_ALL);
- MMIO_RING_GM(RING_START, D_ALL, NULL, NULL);
+ MMIO_RING_DFH(RING_TAIL, D_ALL, F_CMD_ACCESS, NULL, NULL);
+ MMIO_RING_DFH(RING_HEAD, D_ALL, F_CMD_ACCESS, NULL, NULL);
+ MMIO_RING_DFH(RING_CTL, D_ALL, F_CMD_ACCESS, NULL, NULL);
+ MMIO_RING_DFH(RING_ACTHD, D_ALL, F_CMD_ACCESS, NULL, NULL);
+ MMIO_RING_GM_RDR(RING_START, D_ALL, NULL, NULL);
/* RING MODE */
#define RING_REG(base) (base + 0x29c)
- MMIO_RING_DFH(RING_REG, D_ALL, F_MODE_MASK, NULL, ring_mode_mmio_write);
+ MMIO_RING_DFH(RING_REG, D_ALL, F_MODE_MASK | F_CMD_ACCESS, NULL,
+ ring_mode_mmio_write);
#undef RING_REG
- MMIO_RING_DFH(RING_MI_MODE, D_ALL, F_MODE_MASK, NULL, NULL);
- MMIO_RING_DFH(RING_INSTPM, D_ALL, F_MODE_MASK, NULL, NULL);
+ MMIO_RING_DFH(RING_MI_MODE, D_ALL, F_MODE_MASK | F_CMD_ACCESS,
+ NULL, NULL);
+ MMIO_RING_DFH(RING_INSTPM, D_ALL, F_MODE_MASK | F_CMD_ACCESS,
+ NULL, NULL);
MMIO_RING_DFH(RING_TIMESTAMP, D_ALL, F_CMD_ACCESS,
ring_timestamp_mmio_read, NULL);
MMIO_RING_DFH(RING_TIMESTAMP_UDW, D_ALL, F_CMD_ACCESS,
ring_timestamp_mmio_read, NULL);
- MMIO_DFH(GEN7_GT_MODE, D_ALL, F_MODE_MASK, NULL, NULL);
- MMIO_DFH(CACHE_MODE_0_GEN7, D_ALL, F_MODE_MASK, NULL, NULL);
+ MMIO_DFH(GEN7_GT_MODE, D_ALL, F_MODE_MASK | F_CMD_ACCESS, NULL, NULL);
+ MMIO_DFH(CACHE_MODE_0_GEN7, D_ALL, F_MODE_MASK | F_CMD_ACCESS,
+ NULL, NULL);
MMIO_DFH(CACHE_MODE_1, D_ALL, F_MODE_MASK | F_CMD_ACCESS, NULL, NULL);
-
- MMIO_DFH(0x20dc, D_ALL, F_MODE_MASK, NULL, NULL);
- MMIO_DFH(_3D_CHICKEN3, D_ALL, F_MODE_MASK, NULL, NULL);
- MMIO_DFH(0x2088, D_ALL, F_MODE_MASK, NULL, NULL);
- MMIO_DFH(0x20e4, D_ALL, F_MODE_MASK, NULL, NULL);
- MMIO_DFH(0x2470, D_ALL, F_MODE_MASK, NULL, NULL);
- MMIO_D(GAM_ECOCHK, D_ALL);
- MMIO_DFH(GEN7_COMMON_SLICE_CHICKEN1, D_ALL, F_MODE_MASK, NULL, NULL);
+ MMIO_DFH(CACHE_MODE_0, D_ALL, F_MODE_MASK | F_CMD_ACCESS, NULL, NULL);
+ MMIO_DFH(0x2124, D_ALL, F_MODE_MASK | F_CMD_ACCESS, NULL, NULL);
+
+ MMIO_DFH(0x20dc, D_ALL, F_MODE_MASK | F_CMD_ACCESS, NULL, NULL);
+ MMIO_DFH(_3D_CHICKEN3, D_ALL, F_MODE_MASK | F_CMD_ACCESS, NULL, NULL);
+ MMIO_DFH(0x2088, D_ALL, F_MODE_MASK | F_CMD_ACCESS, NULL, NULL);
+ MMIO_DFH(0x20e4, D_ALL, F_MODE_MASK | F_CMD_ACCESS, NULL, NULL);
+ MMIO_DFH(0x2470, D_ALL, F_MODE_MASK | F_CMD_ACCESS, NULL, NULL);
+ MMIO_DFH(GAM_ECOCHK, D_ALL, F_CMD_ACCESS, NULL, NULL);
+ MMIO_DFH(GEN7_COMMON_SLICE_CHICKEN1, D_ALL, F_MODE_MASK | F_CMD_ACCESS,
+ NULL, NULL);
MMIO_DFH(COMMON_SLICE_CHICKEN2, D_ALL, F_MODE_MASK | F_CMD_ACCESS, NULL, NULL);
- MMIO_D(0x9030, D_ALL);
- MMIO_D(0x20a0, D_ALL);
- MMIO_D(0x2420, D_ALL);
- MMIO_D(0x2430, D_ALL);
- MMIO_D(0x2434, D_ALL);
- MMIO_D(0x2438, D_ALL);
- MMIO_D(0x243c, D_ALL);
- MMIO_DFH(0x7018, D_ALL, F_MODE_MASK, NULL, NULL);
+ MMIO_DFH(0x9030, D_ALL, F_CMD_ACCESS, NULL, NULL);
+ MMIO_DFH(0x20a0, D_ALL, F_CMD_ACCESS, NULL, NULL);
+ MMIO_DFH(0x2420, D_ALL, F_CMD_ACCESS, NULL, NULL);
+ MMIO_DFH(0x2430, D_ALL, F_CMD_ACCESS, NULL, NULL);
+ MMIO_DFH(0x2434, D_ALL, F_CMD_ACCESS, NULL, NULL);
+ MMIO_DFH(0x2438, D_ALL, F_CMD_ACCESS, NULL, NULL);
+ MMIO_DFH(0x243c, D_ALL, F_CMD_ACCESS, NULL, NULL);
+ MMIO_DFH(0x7018, D_ALL, F_MODE_MASK | F_CMD_ACCESS, NULL, NULL);
MMIO_DFH(HALF_SLICE_CHICKEN3, D_ALL, F_MODE_MASK | F_CMD_ACCESS, NULL, NULL);
- MMIO_DFH(0xe100, D_ALL, F_MODE_MASK, NULL, NULL);
+ MMIO_DFH(GEN7_HALF_SLICE_CHICKEN1, D_ALL, F_MODE_MASK | F_CMD_ACCESS, NULL, NULL);
/* display */
MMIO_F(0x60220, 0x20, 0, 0, 0, D_ALL, NULL, NULL);
MMIO_D(FORCEWAKE_ACK, D_ALL);
MMIO_D(GEN6_GT_CORE_STATUS, D_ALL);
MMIO_D(GEN6_GT_THREAD_STATUS_REG, D_ALL);
- MMIO_D(GTFIFODBG, D_ALL);
- MMIO_D(GTFIFOCTL, D_ALL);
+ MMIO_DFH(GTFIFODBG, D_ALL, F_CMD_ACCESS, NULL, NULL);
+ MMIO_DFH(GTFIFOCTL, D_ALL, F_CMD_ACCESS, NULL, NULL);
MMIO_DH(FORCEWAKE_MT, D_PRE_SKL, NULL, mul_force_wake_write);
MMIO_DH(FORCEWAKE_ACK_HSW, D_HSW | D_BDW, NULL, NULL);
MMIO_D(ECOBUS, D_ALL);
MMIO_F(0x4f000, 0x90, 0, 0, 0, D_ALL, NULL, NULL);
- MMIO_D(GEN6_PCODE_MAILBOX, D_PRE_SKL);
+ MMIO_D(GEN6_PCODE_MAILBOX, D_PRE_BDW);
MMIO_D(GEN6_PCODE_DATA, D_ALL);
MMIO_D(0x13812c, D_ALL);
MMIO_DH(GEN7_ERR_INT, D_ALL, NULL, NULL);
MMIO_D(0x1a054, D_ALL);
MMIO_D(0x44070, D_ALL);
-
- MMIO_D(0x215c, D_HSW_PLUS);
+ MMIO_DFH(0x215c, D_HSW_PLUS, F_CMD_ACCESS, NULL, NULL);
MMIO_DFH(0x2178, D_ALL, F_CMD_ACCESS, NULL, NULL);
MMIO_DFH(0x217c, D_ALL, F_CMD_ACCESS, NULL, NULL);
MMIO_DFH(0x12178, D_ALL, F_CMD_ACCESS, NULL, NULL);
MMIO_DFH(0x1217c, D_ALL, F_CMD_ACCESS, NULL, NULL);
- MMIO_F(0x2290, 8, 0, 0, 0, D_HSW_PLUS, NULL, NULL);
- MMIO_D(GEN7_OACONTROL, D_HSW);
+ MMIO_F(0x2290, 8, F_CMD_ACCESS, 0, 0, D_HSW_PLUS, NULL, NULL);
+ MMIO_DFH(GEN7_OACONTROL, D_HSW, F_CMD_ACCESS, NULL, NULL);
MMIO_D(0x2b00, D_BDW_PLUS);
MMIO_D(0x2360, D_BDW_PLUS);
- MMIO_F(0x5200, 32, 0, 0, 0, D_ALL, NULL, NULL);
- MMIO_F(0x5240, 32, 0, 0, 0, D_ALL, NULL, NULL);
- MMIO_F(0x5280, 16, 0, 0, 0, D_ALL, NULL, NULL);
+ MMIO_F(0x5200, 32, F_CMD_ACCESS, 0, 0, D_ALL, NULL, NULL);
+ MMIO_F(0x5240, 32, F_CMD_ACCESS, 0, 0, D_ALL, NULL, NULL);
+ MMIO_F(0x5280, 16, F_CMD_ACCESS, 0, 0, D_ALL, NULL, NULL);
MMIO_DFH(0x1c17c, D_BDW_PLUS, F_CMD_ACCESS, NULL, NULL);
MMIO_DFH(0x1c178, D_BDW_PLUS, F_CMD_ACCESS, NULL, NULL);
- MMIO_D(BCS_SWCTRL, D_ALL);
-
- MMIO_F(HS_INVOCATION_COUNT, 8, 0, 0, 0, D_ALL, NULL, NULL);
- MMIO_F(DS_INVOCATION_COUNT, 8, 0, 0, 0, D_ALL, NULL, NULL);
- MMIO_F(IA_VERTICES_COUNT, 8, 0, 0, 0, D_ALL, NULL, NULL);
- MMIO_F(IA_PRIMITIVES_COUNT, 8, 0, 0, 0, D_ALL, NULL, NULL);
- MMIO_F(VS_INVOCATION_COUNT, 8, 0, 0, 0, D_ALL, NULL, NULL);
- MMIO_F(GS_INVOCATION_COUNT, 8, 0, 0, 0, D_ALL, NULL, NULL);
- MMIO_F(GS_PRIMITIVES_COUNT, 8, 0, 0, 0, D_ALL, NULL, NULL);
- MMIO_F(CL_INVOCATION_COUNT, 8, 0, 0, 0, D_ALL, NULL, NULL);
- MMIO_F(CL_PRIMITIVES_COUNT, 8, 0, 0, 0, D_ALL, NULL, NULL);
- MMIO_F(PS_INVOCATION_COUNT, 8, 0, 0, 0, D_ALL, NULL, NULL);
- MMIO_F(PS_DEPTH_COUNT, 8, 0, 0, 0, D_ALL, NULL, NULL);
+ MMIO_DFH(BCS_SWCTRL, D_ALL, F_CMD_ACCESS, NULL, NULL);
+
+ MMIO_F(HS_INVOCATION_COUNT, 8, F_CMD_ACCESS, 0, 0, D_ALL, NULL, NULL);
+ MMIO_F(DS_INVOCATION_COUNT, 8, F_CMD_ACCESS, 0, 0, D_ALL, NULL, NULL);
+ MMIO_F(IA_VERTICES_COUNT, 8, F_CMD_ACCESS, 0, 0, D_ALL, NULL, NULL);
+ MMIO_F(IA_PRIMITIVES_COUNT, 8, F_CMD_ACCESS, 0, 0, D_ALL, NULL, NULL);
+ MMIO_F(VS_INVOCATION_COUNT, 8, F_CMD_ACCESS, 0, 0, D_ALL, NULL, NULL);
+ MMIO_F(GS_INVOCATION_COUNT, 8, F_CMD_ACCESS, 0, 0, D_ALL, NULL, NULL);
+ MMIO_F(GS_PRIMITIVES_COUNT, 8, F_CMD_ACCESS, 0, 0, D_ALL, NULL, NULL);
+ MMIO_F(CL_INVOCATION_COUNT, 8, F_CMD_ACCESS, 0, 0, D_ALL, NULL, NULL);
+ MMIO_F(CL_PRIMITIVES_COUNT, 8, F_CMD_ACCESS, 0, 0, D_ALL, NULL, NULL);
+ MMIO_F(PS_INVOCATION_COUNT, 8, F_CMD_ACCESS, 0, 0, D_ALL, NULL, NULL);
+ MMIO_F(PS_DEPTH_COUNT, 8, F_CMD_ACCESS, 0, 0, D_ALL, NULL, NULL);
MMIO_DH(0x4260, D_BDW_PLUS, NULL, gvt_reg_tlb_control_handler);
MMIO_DH(0x4264, D_BDW_PLUS, NULL, gvt_reg_tlb_control_handler);
MMIO_DH(0x4268, D_BDW_PLUS, NULL, gvt_reg_tlb_control_handler);
MMIO_DH(0x4270, D_BDW_PLUS, NULL, gvt_reg_tlb_control_handler);
MMIO_DFH(0x4094, D_BDW_PLUS, F_CMD_ACCESS, NULL, NULL);
+ MMIO_DFH(ARB_MODE, D_ALL, F_MODE_MASK | F_CMD_ACCESS, NULL, NULL);
+ MMIO_RING_GM_RDR(RING_BBADDR, D_ALL, NULL, NULL);
+ MMIO_DFH(0x2220, D_ALL, F_CMD_ACCESS, NULL, NULL);
+ MMIO_DFH(0x12220, D_ALL, F_CMD_ACCESS, NULL, NULL);
+ MMIO_DFH(0x22220, D_ALL, F_CMD_ACCESS, NULL, NULL);
+ MMIO_RING_DFH(RING_SYNC_1, D_ALL, F_CMD_ACCESS, NULL, NULL);
+ MMIO_RING_DFH(RING_SYNC_0, D_ALL, F_CMD_ACCESS, NULL, NULL);
+ MMIO_DFH(0x22178, D_BDW_PLUS, F_CMD_ACCESS, NULL, NULL);
+ MMIO_DFH(0x1a178, D_BDW_PLUS, F_CMD_ACCESS, NULL, NULL);
+ MMIO_DFH(0x1a17c, D_BDW_PLUS, F_CMD_ACCESS, NULL, NULL);
+ MMIO_DFH(0x2217c, D_BDW_PLUS, F_CMD_ACCESS, NULL, NULL);
return 0;
}
struct drm_i915_private *dev_priv = gvt->dev_priv;
int ret;
- MMIO_DH(RING_IMR(GEN8_BSD2_RING_BASE), D_BDW_PLUS, NULL,
+ MMIO_DFH(RING_IMR(GEN8_BSD2_RING_BASE), D_BDW_PLUS, F_CMD_ACCESS, NULL,
intel_vgpu_reg_imr_handler);
MMIO_DH(GEN8_GT_IMR(0), D_BDW_PLUS, NULL, intel_vgpu_reg_imr_handler);
MMIO_DH(GEN8_MASTER_IRQ, D_BDW_PLUS, NULL,
intel_vgpu_reg_master_irq_handler);
- MMIO_D(RING_HWSTAM(GEN8_BSD2_RING_BASE), D_BDW_PLUS);
- MMIO_D(0x1c134, D_BDW_PLUS);
-
- MMIO_D(RING_TAIL(GEN8_BSD2_RING_BASE), D_BDW_PLUS);
- MMIO_D(RING_HEAD(GEN8_BSD2_RING_BASE), D_BDW_PLUS);
- MMIO_GM(RING_START(GEN8_BSD2_RING_BASE), D_BDW_PLUS, NULL, NULL);
- MMIO_D(RING_CTL(GEN8_BSD2_RING_BASE), D_BDW_PLUS);
- MMIO_D(RING_ACTHD(GEN8_BSD2_RING_BASE), D_BDW_PLUS);
- MMIO_D(RING_ACTHD_UDW(GEN8_BSD2_RING_BASE), D_BDW_PLUS);
- MMIO_DFH(0x1c29c, D_BDW_PLUS, F_MODE_MASK, NULL, ring_mode_mmio_write);
- MMIO_DFH(RING_MI_MODE(GEN8_BSD2_RING_BASE), D_BDW_PLUS, F_MODE_MASK,
- NULL, NULL);
- MMIO_DFH(RING_INSTPM(GEN8_BSD2_RING_BASE), D_BDW_PLUS, F_MODE_MASK,
- NULL, NULL);
+ MMIO_DFH(RING_HWSTAM(GEN8_BSD2_RING_BASE), D_BDW_PLUS,
+ F_CMD_ACCESS, NULL, NULL);
+ MMIO_DFH(0x1c134, D_BDW_PLUS, F_CMD_ACCESS, NULL, NULL);
+
+ MMIO_DFH(RING_TAIL(GEN8_BSD2_RING_BASE), D_BDW_PLUS, F_CMD_ACCESS,
+ NULL, NULL);
+ MMIO_DFH(RING_HEAD(GEN8_BSD2_RING_BASE), D_BDW_PLUS,
+ F_CMD_ACCESS, NULL, NULL);
+ MMIO_GM_RDR(RING_START(GEN8_BSD2_RING_BASE), D_BDW_PLUS, NULL, NULL);
+ MMIO_DFH(RING_CTL(GEN8_BSD2_RING_BASE), D_BDW_PLUS, F_CMD_ACCESS,
+ NULL, NULL);
+ MMIO_DFH(RING_ACTHD(GEN8_BSD2_RING_BASE), D_BDW_PLUS,
+ F_CMD_ACCESS, NULL, NULL);
+ MMIO_DFH(RING_ACTHD_UDW(GEN8_BSD2_RING_BASE), D_BDW_PLUS,
+ F_CMD_ACCESS, NULL, NULL);
+ MMIO_DFH(0x1c29c, D_BDW_PLUS, F_MODE_MASK | F_CMD_ACCESS, NULL,
+ ring_mode_mmio_write);
+ MMIO_DFH(RING_MI_MODE(GEN8_BSD2_RING_BASE), D_BDW_PLUS,
+ F_MODE_MASK | F_CMD_ACCESS, NULL, NULL);
+ MMIO_DFH(RING_INSTPM(GEN8_BSD2_RING_BASE), D_BDW_PLUS,
+ F_MODE_MASK | F_CMD_ACCESS, NULL, NULL);
MMIO_DFH(RING_TIMESTAMP(GEN8_BSD2_RING_BASE), D_BDW_PLUS, F_CMD_ACCESS,
ring_timestamp_mmio_read, NULL);
- MMIO_RING_D(RING_ACTHD_UDW, D_BDW_PLUS);
+ MMIO_RING_DFH(RING_ACTHD_UDW, D_BDW_PLUS, F_CMD_ACCESS, NULL, NULL);
#define RING_REG(base) (base + 0xd0)
MMIO_RING_F(RING_REG, 4, F_RO, 0,
#undef RING_REG
#define RING_REG(base) (base + 0x234)
- MMIO_RING_F(RING_REG, 8, F_RO, 0, ~0, D_BDW_PLUS, NULL, NULL);
- MMIO_F(RING_REG(GEN8_BSD2_RING_BASE), 4, F_RO, 0, ~0LL, D_BDW_PLUS, NULL, NULL);
+ MMIO_RING_F(RING_REG, 8, F_RO | F_CMD_ACCESS, 0, ~0, D_BDW_PLUS,
+ NULL, NULL);
+ MMIO_F(RING_REG(GEN8_BSD2_RING_BASE), 4, F_RO | F_CMD_ACCESS, 0,
+ ~0LL, D_BDW_PLUS, NULL, NULL);
#undef RING_REG
#define RING_REG(base) (base + 0x244)
- MMIO_RING_D(RING_REG, D_BDW_PLUS);
- MMIO_D(RING_REG(GEN8_BSD2_RING_BASE), D_BDW_PLUS);
+ MMIO_RING_DFH(RING_REG, D_BDW_PLUS, F_CMD_ACCESS, NULL, NULL);
+ MMIO_DFH(RING_REG(GEN8_BSD2_RING_BASE), D_BDW_PLUS, F_CMD_ACCESS,
+ NULL, NULL);
#undef RING_REG
#define RING_REG(base) (base + 0x370)
MMIO_D(GEN7_MISCCPCTL, D_BDW_PLUS);
MMIO_D(0x1c054, D_BDW_PLUS);
+ MMIO_DH(GEN6_PCODE_MAILBOX, D_BDW_PLUS, NULL, mailbox_write);
+
MMIO_D(GEN8_PRIVATE_PAT_LO, D_BDW_PLUS);
MMIO_D(GEN8_PRIVATE_PAT_HI, D_BDW_PLUS);
MMIO_F(RING_REG(GEN8_BSD2_RING_BASE), 32, 0, 0, 0, D_BDW_PLUS, NULL, NULL);
#undef RING_REG
- MMIO_RING_GM(RING_HWS_PGA, D_BDW_PLUS, NULL, NULL);
- MMIO_GM(0x1c080, D_BDW_PLUS, NULL, NULL);
+ MMIO_RING_GM_RDR(RING_HWS_PGA, D_BDW_PLUS, NULL, NULL);
+ MMIO_GM_RDR(RING_HWS_PGA(GEN8_BSD2_RING_BASE), D_BDW_PLUS, NULL, NULL);
MMIO_DFH(HDC_CHICKEN0, D_BDW_PLUS, F_MODE_MASK | F_CMD_ACCESS, NULL, NULL);
- MMIO_D(CHICKEN_PIPESL_1(PIPE_A), D_BDW);
- MMIO_D(CHICKEN_PIPESL_1(PIPE_B), D_BDW);
- MMIO_D(CHICKEN_PIPESL_1(PIPE_C), D_BDW);
+ MMIO_D(CHICKEN_PIPESL_1(PIPE_A), D_BDW_PLUS);
+ MMIO_D(CHICKEN_PIPESL_1(PIPE_B), D_BDW_PLUS);
+ MMIO_D(CHICKEN_PIPESL_1(PIPE_C), D_BDW_PLUS);
MMIO_D(WM_MISC, D_BDW);
MMIO_D(BDW_EDP_PSR_BASE, D_BDW);
MMIO_D(GEN8_EU_DISABLE1, D_BDW_PLUS);
MMIO_D(GEN8_EU_DISABLE2, D_BDW_PLUS);
- MMIO_D(0xfdc, D_BDW);
- MMIO_DFH(GEN8_ROW_CHICKEN, D_BDW_PLUS, F_CMD_ACCESS, NULL, NULL);
- MMIO_D(GEN7_ROW_CHICKEN2, D_BDW_PLUS);
- MMIO_D(GEN8_UCGCTL6, D_BDW_PLUS);
+ MMIO_D(0xfdc, D_BDW_PLUS);
+ MMIO_DFH(GEN8_ROW_CHICKEN, D_BDW_PLUS, F_MODE_MASK | F_CMD_ACCESS,
+ NULL, NULL);
+ MMIO_DFH(GEN7_ROW_CHICKEN2, D_BDW_PLUS, F_MODE_MASK | F_CMD_ACCESS,
+ NULL, NULL);
+ MMIO_DFH(GEN8_UCGCTL6, D_BDW_PLUS, F_CMD_ACCESS, NULL, NULL);
- MMIO_D(0xb1f0, D_BDW);
- MMIO_D(0xb1c0, D_BDW);
+ MMIO_DFH(0xb1f0, D_BDW, F_CMD_ACCESS, NULL, NULL);
+ MMIO_DFH(0xb1c0, D_BDW, F_CMD_ACCESS, NULL, NULL);
MMIO_DFH(GEN8_L3SQCREG4, D_BDW_PLUS, F_CMD_ACCESS, NULL, NULL);
- MMIO_D(0xb100, D_BDW);
- MMIO_D(0xb10c, D_BDW);
+ MMIO_DFH(0xb100, D_BDW, F_CMD_ACCESS, NULL, NULL);
+ MMIO_DFH(0xb10c, D_BDW, F_CMD_ACCESS, NULL, NULL);
MMIO_D(0xb110, D_BDW);
- MMIO_DFH(0x24d0, D_BDW_PLUS, F_CMD_ACCESS, NULL, NULL);
- MMIO_DFH(0x24d4, D_BDW_PLUS, F_CMD_ACCESS, NULL, NULL);
- MMIO_DFH(0x24d8, D_BDW_PLUS, F_CMD_ACCESS, NULL, NULL);
- MMIO_DFH(0x24dc, D_BDW_PLUS, F_CMD_ACCESS, NULL, NULL);
+ MMIO_F(0x24d0, 48, F_CMD_ACCESS, 0, 0, D_BDW_PLUS,
+ NULL, force_nonpriv_write);
+
+ MMIO_D(0x22040, D_BDW_PLUS);
+ MMIO_D(0x44484, D_BDW_PLUS);
+ MMIO_D(0x4448c, D_BDW_PLUS);
- MMIO_D(0x83a4, D_BDW);
+ MMIO_DFH(0x83a4, D_BDW, F_CMD_ACCESS, NULL, NULL);
MMIO_D(GEN8_L3_LRA_1_GPGPU, D_BDW_PLUS);
- MMIO_D(0x8430, D_BDW);
+ MMIO_DFH(0x8430, D_BDW, F_CMD_ACCESS, NULL, NULL);
MMIO_D(0x110000, D_BDW_PLUS);
MMIO_DFH(0xe194, D_BDW_PLUS, F_MODE_MASK | F_CMD_ACCESS, NULL, NULL);
MMIO_DFH(0xe188, D_BDW_PLUS, F_MODE_MASK | F_CMD_ACCESS, NULL, NULL);
MMIO_DFH(HALF_SLICE_CHICKEN2, D_BDW_PLUS, F_MODE_MASK | F_CMD_ACCESS, NULL, NULL);
- MMIO_DFH(0x2580, D_BDW_PLUS, F_MODE_MASK, NULL, NULL);
-
- MMIO_D(0x2248, D_BDW);
-
+ MMIO_DFH(0x2580, D_BDW_PLUS, F_MODE_MASK | F_CMD_ACCESS, NULL, NULL);
+
+ MMIO_DFH(0x2248, D_BDW, F_CMD_ACCESS, NULL, NULL);
+
+ MMIO_DFH(0xe220, D_BDW_PLUS, F_CMD_ACCESS, NULL, NULL);
+ MMIO_DFH(0xe230, D_BDW_PLUS, F_CMD_ACCESS, NULL, NULL);
+ MMIO_DFH(0xe240, D_BDW_PLUS, F_CMD_ACCESS, NULL, NULL);
+ MMIO_DFH(0xe260, D_BDW_PLUS, F_CMD_ACCESS, NULL, NULL);
+ MMIO_DFH(0xe270, D_BDW_PLUS, F_CMD_ACCESS, NULL, NULL);
+ MMIO_DFH(0xe280, D_BDW_PLUS, F_CMD_ACCESS, NULL, NULL);
+ MMIO_DFH(0xe2a0, D_BDW_PLUS, F_CMD_ACCESS, NULL, NULL);
+ MMIO_DFH(0xe2b0, D_BDW_PLUS, F_CMD_ACCESS, NULL, NULL);
+ MMIO_DFH(0xe2c0, D_BDW_PLUS, F_CMD_ACCESS, NULL, NULL);
return 0;
}
MMIO_D(HSW_PWR_WELL_BIOS, D_SKL);
MMIO_DH(HSW_PWR_WELL_DRIVER, D_SKL, NULL, skl_power_well_ctl_write);
- MMIO_DH(GEN6_PCODE_MAILBOX, D_SKL, NULL, mailbox_write);
MMIO_D(0xa210, D_SKL_PLUS);
MMIO_D(GEN9_MEDIA_PG_IDLE_HYSTERESIS, D_SKL_PLUS);
MMIO_D(GEN9_RENDER_PG_IDLE_HYSTERESIS, D_SKL_PLUS);
MMIO_F(0xb020, 0x80, F_CMD_ACCESS, 0, 0, D_SKL, NULL, NULL);
MMIO_D(0xd08, D_SKL);
- MMIO_D(0x20e0, D_SKL);
- MMIO_D(0x20ec, D_SKL);
+ MMIO_DFH(0x20e0, D_SKL, F_MODE_MASK, NULL, NULL);
+ MMIO_DFH(0x20ec, D_SKL, F_MODE_MASK | F_CMD_ACCESS, NULL, NULL);
/* TRTT */
- MMIO_D(0x4de0, D_SKL);
- MMIO_D(0x4de4, D_SKL);
- MMIO_D(0x4de8, D_SKL);
- MMIO_D(0x4dec, D_SKL);
- MMIO_D(0x4df0, D_SKL);
- MMIO_DH(0x4df4, D_SKL, NULL, gen9_trtte_write);
+ MMIO_DFH(0x4de0, D_SKL, F_CMD_ACCESS, NULL, NULL);
+ MMIO_DFH(0x4de4, D_SKL, F_CMD_ACCESS, NULL, NULL);
+ MMIO_DFH(0x4de8, D_SKL, F_CMD_ACCESS, NULL, NULL);
+ MMIO_DFH(0x4dec, D_SKL, F_CMD_ACCESS, NULL, NULL);
+ MMIO_DFH(0x4df0, D_SKL, F_CMD_ACCESS, NULL, NULL);
+ MMIO_DFH(0x4df4, D_SKL, F_CMD_ACCESS, NULL, gen9_trtte_write);
MMIO_DH(0x4dfc, D_SKL, NULL, gen9_trtt_chicken_write);
MMIO_D(0x45008, D_SKL);
MMIO_D(0x65f08, D_SKL);
MMIO_D(0x320f0, D_SKL);
- MMIO_D(_REG_VCS2_EXCC, D_SKL);
+ MMIO_DFH(_REG_VCS2_EXCC, D_SKL, F_CMD_ACCESS, NULL, NULL);
MMIO_D(0x70034, D_SKL);
MMIO_D(0x71034, D_SKL);
MMIO_D(0x72034, D_SKL);
MMIO_D(_PLANE_KEYMSK_1(PIPE_C), D_SKL);
MMIO_D(0x44500, D_SKL);
+ MMIO_DFH(GEN9_CSFE_CHICKEN1_RCS, D_SKL_PLUS, F_CMD_ACCESS, NULL, NULL);
+ MMIO_DFH(GEN8_HDC_CHICKEN1, D_SKL, F_MODE_MASK | F_CMD_ACCESS,
+ NULL, NULL);
return 0;
}
struct device *dev = &vgpu->gvt->dev_priv->drm.pdev->dev;
dma_addr_t daddr;
- page = pfn_to_page(pfn);
- if (is_error_page(page))
+ if (unlikely(!pfn_valid(pfn)))
return -EFAULT;
+ page = pfn_to_page(pfn);
daddr = dma_map_page(dev, page, 0, PAGE_SIZE,
PCI_DMA_BIDIRECTIONAL);
if (dma_mapping_error(dev, daddr))
return 0;
return sprintf(buf, "low_gm_size: %dMB\nhigh_gm_size: %dMB\n"
- "fence: %d\n",
- BYTES_TO_MB(type->low_gm_size),
- BYTES_TO_MB(type->high_gm_size),
- type->fence);
+ "fence: %d\nresolution: %s\n",
+ BYTES_TO_MB(type->low_gm_size),
+ BYTES_TO_MB(type->high_gm_size),
+ type->fence, vgpu_edid_str(type->resolution));
}
static MDEV_TYPE_ATTR_RO(available_instances);
(reg >= gvt->device_info.gtt_start_offset \
&& reg < gvt->device_info.gtt_start_offset + gvt_ggtt_sz(gvt))
+static void failsafe_emulate_mmio_rw(struct intel_vgpu *vgpu, uint64_t pa,
+ void *p_data, unsigned int bytes, bool read)
+{
+ struct intel_gvt *gvt = NULL;
+ void *pt = NULL;
+ unsigned int offset = 0;
+
+ if (!vgpu || !p_data)
+ return;
+
+ gvt = vgpu->gvt;
+ mutex_lock(&gvt->lock);
+ offset = intel_vgpu_gpa_to_mmio_offset(vgpu, pa);
+ if (reg_is_mmio(gvt, offset)) {
+ if (read)
+ intel_vgpu_default_mmio_read(vgpu, offset, p_data,
+ bytes);
+ else
+ intel_vgpu_default_mmio_write(vgpu, offset, p_data,
+ bytes);
+ } else if (reg_is_gtt(gvt, offset) &&
+ vgpu->gtt.ggtt_mm->virtual_page_table) {
+ offset -= gvt->device_info.gtt_start_offset;
+ pt = vgpu->gtt.ggtt_mm->virtual_page_table + offset;
+ if (read)
+ memcpy(p_data, pt, bytes);
+ else
+ memcpy(pt, p_data, bytes);
+
+ } else if (atomic_read(&vgpu->gtt.n_write_protected_guest_page)) {
+ struct intel_vgpu_guest_page *gp;
+
+ /* Since we enter the failsafe mode early during guest boot,
+ * guest may not have chance to set up its ppgtt table, so
+ * there should not be any wp pages for guest. Keep the wp
+ * related code here in case we need to handle it in furture.
+ */
+ gp = intel_vgpu_find_guest_page(vgpu, pa >> PAGE_SHIFT);
+ if (gp) {
+ /* remove write protection to prevent furture traps */
+ intel_vgpu_clean_guest_page(vgpu, gp);
+ if (read)
+ intel_gvt_hypervisor_read_gpa(vgpu, pa,
+ p_data, bytes);
+ else
+ intel_gvt_hypervisor_write_gpa(vgpu, pa,
+ p_data, bytes);
+ }
+ }
+ mutex_unlock(&gvt->lock);
+}
+
/**
* intel_vgpu_emulate_mmio_read - emulate MMIO read
* @vgpu: a vGPU
unsigned int offset = 0;
int ret = -EINVAL;
+
+ if (vgpu->failsafe) {
+ failsafe_emulate_mmio_rw(vgpu, pa, p_data, bytes, true);
+ return 0;
+ }
mutex_lock(&gvt->lock);
if (atomic_read(&vgpu->gtt.n_write_protected_guest_page)) {
u32 old_vreg = 0, old_sreg = 0;
int ret = -EINVAL;
+ if (vgpu->failsafe) {
+ failsafe_emulate_mmio_rw(vgpu, pa, p_data, bytes, false);
+ return 0;
+ }
+
mutex_lock(&gvt->lock);
if (atomic_read(&vgpu->gtt.n_write_protected_guest_page)) {
mmio = intel_gvt_find_mmio_info(gvt, rounddown(offset, 4));
if (!mmio && !vgpu->mmio.disable_warn_untrack)
- gvt_err("vgpu%d: write untracked MMIO %x len %d val %x\n",
+ gvt_dbg_mmio("vgpu%d: write untracked MMIO %x len %d val %x\n",
vgpu->id, offset, bytes, *(u32 *)p_data);
if (!intel_gvt_mmio_is_unalign(gvt, offset)) {
/* set the bit 0:2(Core C-State ) to C0 */
vgpu_vreg(vgpu, GEN6_GT_CORE_STATUS) = 0;
+
+ vgpu->mmio.disable_warn_untrack = false;
}
/**
static int init_vgpu_opregion(struct intel_vgpu *vgpu, u32 gpa)
{
- void __iomem *host_va = vgpu->gvt->opregion.opregion_va;
u8 *buf;
int i;
if (!vgpu_opregion(vgpu)->va)
return -ENOMEM;
- memcpy_fromio(vgpu_opregion(vgpu)->va, host_va,
- INTEL_GVT_OPREGION_SIZE);
+ memcpy(vgpu_opregion(vgpu)->va, vgpu->gvt->opregion.opregion_va,
+ INTEL_GVT_OPREGION_SIZE);
for (i = 0; i < INTEL_GVT_OPREGION_PAGES; i++)
vgpu_opregion(vgpu)->gfn[i] = (gpa >> PAGE_SHIFT) + i;
{RCS, _MMIO(0x24d4), 0, false},
{RCS, _MMIO(0x24d8), 0, false},
{RCS, _MMIO(0x24dc), 0, false},
+ {RCS, _MMIO(0x24e0), 0, false},
+ {RCS, _MMIO(0x24e4), 0, false},
+ {RCS, _MMIO(0x24e8), 0, false},
+ {RCS, _MMIO(0x24ec), 0, false},
+ {RCS, _MMIO(0x24f0), 0, false},
+ {RCS, _MMIO(0x24f4), 0, false},
+ {RCS, _MMIO(0x24f8), 0, false},
+ {RCS, _MMIO(0x24fc), 0, false},
{RCS, _MMIO(0x7004), 0xffff, true},
{RCS, _MMIO(0x7008), 0xffff, true},
{RCS, _MMIO(0x7000), 0xffff, true},
{RCS, _MMIO(0x24d4), 0, false},
{RCS, _MMIO(0x24d8), 0, false},
{RCS, _MMIO(0x24dc), 0, false},
+ {RCS, _MMIO(0x24e0), 0, false},
+ {RCS, _MMIO(0x24e4), 0, false},
+ {RCS, _MMIO(0x24e8), 0, false},
+ {RCS, _MMIO(0x24ec), 0, false},
+ {RCS, _MMIO(0x24f0), 0, false},
+ {RCS, _MMIO(0x24f4), 0, false},
+ {RCS, _MMIO(0x24f8), 0, false},
+ {RCS, _MMIO(0x24fc), 0, false},
{RCS, _MMIO(0x7004), 0xffff, true},
{RCS, _MMIO(0x7008), 0xffff, true},
{RCS, _MMIO(0x7000), 0xffff, true},
struct intel_vgpu_workload *workload =
scheduler->current_workload[req->engine->id];
+ if (unlikely(!workload))
+ return NOTIFY_OK;
+
switch (action) {
case INTEL_CONTEXT_SCHEDULE_IN:
intel_gvt_load_render_mmio(workload->vgpu,
case INTEL_CONTEXT_SCHEDULE_OUT:
intel_gvt_restore_render_mmio(workload->vgpu,
workload->ring_id);
+ /* If the status is -EINPROGRESS means this workload
+ * doesn't meet any issue during dispatching so when
+ * get the SCHEDULE_OUT set the status to be zero for
+ * good. If the status is NOT -EINPROGRESS means there
+ * is something wrong happened during dispatching and
+ * the status should not be set to zero
+ */
+ if (workload->status == -EINPROGRESS)
+ workload->status = 0;
atomic_set(&workload->shadow_ctx_active, 0);
break;
default:
workload = scheduler->current_workload[ring_id];
vgpu = workload->vgpu;
- if (!workload->status && !vgpu->resetting) {
+ /* For the workload w/ request, needs to wait for the context
+ * switch to make sure request is completed.
+ * For the workload w/o request, directly complete the workload.
+ */
+ if (workload->req) {
wait_event(workload->shadow_ctx_status_wq,
!atomic_read(&workload->shadow_ctx_active));
- update_guest_context(workload);
+ i915_gem_request_put(fetch_and_zero(&workload->req));
+
+ if (!workload->status && !vgpu->resetting) {
+ update_guest_context(workload);
- for_each_set_bit(event, workload->pending_events,
- INTEL_GVT_EVENT_MAX)
- intel_vgpu_trigger_virtual_event(vgpu, event);
+ for_each_set_bit(event, workload->pending_events,
+ INTEL_GVT_EVENT_MAX)
+ intel_vgpu_trigger_virtual_event(vgpu, event);
+ }
}
gvt_dbg_sched("ring id %d complete workload %p status %d\n",
int ring_id = p->ring_id;
struct intel_gvt_workload_scheduler *scheduler = &gvt->scheduler;
struct intel_vgpu_workload *workload = NULL;
- long lret;
int ret;
bool need_force_wake = IS_SKYLAKE(gvt->dev_priv);
DEFINE_WAIT_FUNC(wait, woken_wake_function);
gvt_dbg_sched("ring id %d wait workload %p\n",
workload->ring_id, workload);
-
- lret = i915_wait_request(workload->req,
+retry:
+ i915_wait_request(workload->req,
0, MAX_SCHEDULE_TIMEOUT);
- if (lret < 0) {
- workload->status = lret;
- gvt_err("fail to wait workload, skip\n");
- } else {
- workload->status = 0;
+ /* I915 has replay mechanism and a request will be replayed
+ * if there is i915 reset. So the seqno will be updated anyway.
+ * If the seqno is not updated yet after waiting, which means
+ * the replay may still be in progress and we can wait again.
+ */
+ if (!i915_gem_request_completed(workload->req)) {
+ gvt_dbg_sched("workload %p not completed, wait again\n",
+ workload);
+ goto retry;
}
complete:
gvt_dbg_sched("will complete workload %p, status: %d\n",
workload, workload->status);
- if (workload->req)
- i915_gem_request_put(fetch_and_zero(&workload->req));
-
complete_current_workload(gvt, ring_id);
if (need_force_wake)
WARN_ON(sizeof(struct vgt_if) != VGT_PVINFO_SIZE);
}
+static struct {
+ unsigned int low_mm;
+ unsigned int high_mm;
+ unsigned int fence;
+ enum intel_vgpu_edid edid;
+ char *name;
+} vgpu_types[] = {
+/* Fixed vGPU type table */
+ { MB_TO_BYTES(64), MB_TO_BYTES(512), 4, GVT_EDID_1024_768, "8" },
+ { MB_TO_BYTES(128), MB_TO_BYTES(512), 4, GVT_EDID_1920_1200, "4" },
+ { MB_TO_BYTES(256), MB_TO_BYTES(1024), 4, GVT_EDID_1920_1200, "2" },
+ { MB_TO_BYTES(512), MB_TO_BYTES(2048), 4, GVT_EDID_1920_1200, "1" },
+};
+
/**
* intel_gvt_init_vgpu_types - initialize vGPU type list
* @gvt : GVT device
unsigned int min_low;
/* vGPU type name is defined as GVTg_Vx_y which contains
- * physical GPU generation type and 'y' means maximum vGPU
- * instances user can create on one physical GPU for this
- * type.
+ * physical GPU generation type (e.g V4 as BDW server, V5 as
+ * SKL server).
*
* Depend on physical SKU resource, might see vGPU types like
* GVTg_V4_8, GVTg_V4_4, GVTg_V4_2, etc. We can create
*/
low_avail = gvt_aperture_sz(gvt) - HOST_LOW_GM_SIZE;
high_avail = gvt_hidden_sz(gvt) - HOST_HIGH_GM_SIZE;
- num_types = 4;
+ num_types = sizeof(vgpu_types) / sizeof(vgpu_types[0]);
gvt->types = kzalloc(num_types * sizeof(struct intel_vgpu_type),
GFP_KERNEL);
min_low = MB_TO_BYTES(32);
for (i = 0; i < num_types; ++i) {
- if (low_avail / min_low == 0)
+ if (low_avail / vgpu_types[i].low_mm == 0)
break;
- gvt->types[i].low_gm_size = min_low;
- gvt->types[i].high_gm_size = max((min_low<<3), MB_TO_BYTES(384U));
- gvt->types[i].fence = 4;
- gvt->types[i].max_instance = min(low_avail / min_low,
- high_avail / gvt->types[i].high_gm_size);
- gvt->types[i].avail_instance = gvt->types[i].max_instance;
+
+ gvt->types[i].low_gm_size = vgpu_types[i].low_mm;
+ gvt->types[i].high_gm_size = vgpu_types[i].high_mm;
+ gvt->types[i].fence = vgpu_types[i].fence;
+ gvt->types[i].resolution = vgpu_types[i].edid;
+ gvt->types[i].avail_instance = min(low_avail / vgpu_types[i].low_mm,
+ high_avail / vgpu_types[i].high_mm);
if (IS_GEN8(gvt->dev_priv))
- sprintf(gvt->types[i].name, "GVTg_V4_%u",
- gvt->types[i].max_instance);
+ sprintf(gvt->types[i].name, "GVTg_V4_%s",
+ vgpu_types[i].name);
else if (IS_GEN9(gvt->dev_priv))
- sprintf(gvt->types[i].name, "GVTg_V5_%u",
- gvt->types[i].max_instance);
+ sprintf(gvt->types[i].name, "GVTg_V5_%s",
+ vgpu_types[i].name);
- min_low <<= 1;
- gvt_dbg_core("type[%d]: %s max %u avail %u low %u high %u fence %u\n",
- i, gvt->types[i].name, gvt->types[i].max_instance,
+ gvt_dbg_core("type[%d]: %s avail %u low %u high %u fence %u res %s\n",
+ i, gvt->types[i].name,
gvt->types[i].avail_instance,
gvt->types[i].low_gm_size,
- gvt->types[i].high_gm_size, gvt->types[i].fence);
+ gvt->types[i].high_gm_size, gvt->types[i].fence,
+ vgpu_edid_str(gvt->types[i].resolution));
}
gvt->num_types = i;
{
int i;
unsigned int low_gm_avail, high_gm_avail, fence_avail;
- unsigned int low_gm_min, high_gm_min, fence_min, total_min;
+ unsigned int low_gm_min, high_gm_min, fence_min;
/* Need to depend on maxium hw resource size but keep on
* static config for now.
low_gm_min = low_gm_avail / gvt->types[i].low_gm_size;
high_gm_min = high_gm_avail / gvt->types[i].high_gm_size;
fence_min = fence_avail / gvt->types[i].fence;
- total_min = min(min(low_gm_min, high_gm_min), fence_min);
- gvt->types[i].avail_instance = min(gvt->types[i].max_instance,
- total_min);
+ gvt->types[i].avail_instance = min(min(low_gm_min, high_gm_min),
+ fence_min);
- gvt_dbg_core("update type[%d]: %s max %u avail %u low %u high %u fence %u\n",
- i, gvt->types[i].name, gvt->types[i].max_instance,
+ gvt_dbg_core("update type[%d]: %s avail %u low %u high %u fence %u\n",
+ i, gvt->types[i].name,
gvt->types[i].avail_instance, gvt->types[i].low_gm_size,
gvt->types[i].high_gm_size, gvt->types[i].fence);
}
if (ret)
goto out_detach_hypervisor_vgpu;
- ret = intel_vgpu_init_display(vgpu);
+ ret = intel_vgpu_init_display(vgpu, param->resolution);
if (ret)
goto out_clean_gtt;
param.low_gm_sz = type->low_gm_size;
param.high_gm_sz = type->high_gm_size;
param.fence_sz = type->fence;
+ param.resolution = type->resolution;
/* XXX current param based on MB */
param.low_gm_sz = BYTES_TO_MB(param.low_gm_sz);
populate_pvinfo_page(vgpu);
intel_vgpu_reset_display(vgpu);
- if (dmlr)
+ if (dmlr) {
intel_vgpu_reset_cfg_space(vgpu);
+ /* only reset the failsafe mode when dmlr reset */
+ vgpu->failsafe = false;
+ vgpu->pv_notified = false;
+ }
}
vgpu->resetting = false;
if (!obj->base.filp)
return -ENODEV;
- ret = obj->base.filp->f_op->mmap(obj->base.filp, vma);
+ ret = call_mmap(obj->base.filp, vma);
if (ret)
return ret;
#include <linux/prefetch.h>
#include <linux/dma-fence-array.h>
+#include <linux/sched.h>
+#include <linux/sched/clock.h>
+#include <linux/sched/signal.h>
#include "i915_drv.h"
#include <linux/mmu_notifier.h>
#include <linux/mempolicy.h>
#include <linux/swap.h>
+#include <linux/sched/mm.h>
struct i915_mm_struct {
struct mm_struct *mm;
*/
#include <linux/kthread.h>
+#include <uapi/linux/sched/types.h>
#include "i915_drv.h"
#include <linux/async.h>
#include <linux/i2c.h>
#include <linux/hdmi.h>
+#include <linux/sched/clock.h>
#include <drm/i915_drm.h>
#include "i915_drv.h"
#include <drm/drm_crtc.h>
switch (format) {
case DRM_FORMAT_RGB565:
dev_dbg(drm->dev, "Setting up RGB565 mode\n");
- ctrl |= CTRL_SET_BUS_WIDTH(STMLCDIF_16BIT);
ctrl |= CTRL_SET_WORD_LENGTH(0);
ctrl1 |= CTRL1_SET_BYTE_PACKAGING(0xf);
break;
case DRM_FORMAT_XRGB8888:
dev_dbg(drm->dev, "Setting up XRGB8888 mode\n");
- ctrl |= CTRL_SET_BUS_WIDTH(STMLCDIF_24BIT);
ctrl |= CTRL_SET_WORD_LENGTH(3);
/* Do not use packed pixels = one pixel per word instead. */
ctrl1 |= CTRL1_SET_BYTE_PACKAGING(0x7);
return 0;
}
+static void mxsfb_set_bus_fmt(struct mxsfb_drm_private *mxsfb)
+{
+ struct drm_crtc *crtc = &mxsfb->pipe.crtc;
+ struct drm_device *drm = crtc->dev;
+ u32 bus_format = MEDIA_BUS_FMT_RGB888_1X24;
+ u32 reg;
+
+ reg = readl(mxsfb->base + LCDC_CTRL);
+
+ if (mxsfb->connector.display_info.num_bus_formats)
+ bus_format = mxsfb->connector.display_info.bus_formats[0];
+
+ reg &= ~CTRL_BUS_WIDTH_MASK;
+ switch (bus_format) {
+ case MEDIA_BUS_FMT_RGB565_1X16:
+ reg |= CTRL_SET_BUS_WIDTH(STMLCDIF_16BIT);
+ break;
+ case MEDIA_BUS_FMT_RGB666_1X18:
+ reg |= CTRL_SET_BUS_WIDTH(STMLCDIF_18BIT);
+ break;
+ case MEDIA_BUS_FMT_RGB888_1X24:
+ reg |= CTRL_SET_BUS_WIDTH(STMLCDIF_24BIT);
+ break;
+ default:
+ dev_err(drm->dev, "Unknown media bus format %d\n", bus_format);
+ break;
+ }
+ writel(reg, mxsfb->base + LCDC_CTRL);
+}
+
static void mxsfb_enable_controller(struct mxsfb_drm_private *mxsfb)
{
u32 reg;
vdctrl0 |= VDCTRL0_HSYNC_ACT_HIGH;
if (m->flags & DRM_MODE_FLAG_PVSYNC)
vdctrl0 |= VDCTRL0_VSYNC_ACT_HIGH;
- if (bus_flags & DRM_BUS_FLAG_DE_HIGH)
+ /* Make sure Data Enable is high active by default */
+ if (!(bus_flags & DRM_BUS_FLAG_DE_LOW))
vdctrl0 |= VDCTRL0_ENABLE_ACT_HIGH;
- if (bus_flags & DRM_BUS_FLAG_PIXDATA_NEGEDGE)
+ /*
+ * DRM_BUS_FLAG_PIXDATA_ defines are controller centric,
+ * controllers VDCTRL0_DOTCLK is display centric.
+ * Drive on positive edge -> display samples on falling edge
+ * DRM_BUS_FLAG_PIXDATA_POSEDGE -> VDCTRL0_DOTCLK_ACT_FALLING
+ */
+ if (bus_flags & DRM_BUS_FLAG_PIXDATA_POSEDGE)
vdctrl0 |= VDCTRL0_DOTCLK_ACT_FALLING;
writel(vdctrl0, mxsfb->base + LCDC_VDCTRL0);
+ mxsfb_set_bus_fmt(mxsfb);
+
/* Frame length in lines. */
writel(m->crtc_vtotal, mxsfb->base + LCDC_VDCTRL1);
VDCTRL2_SET_HSYNC_PERIOD(m->crtc_htotal),
mxsfb->base + LCDC_VDCTRL2);
- writel(SET_HOR_WAIT_CNT(m->crtc_hblank_end - m->crtc_hsync_end) |
- SET_VERT_WAIT_CNT(m->crtc_vblank_end - m->crtc_vsync_end),
+ writel(SET_HOR_WAIT_CNT(m->crtc_htotal - m->crtc_hsync_start) |
+ SET_VERT_WAIT_CNT(m->crtc_vtotal - m->crtc_vsync_start),
mxsfb->base + LCDC_VDCTRL3);
writel(SET_DOTCLK_H_VALID_DATA_CNT(m->hdisplay),
{
struct mxsfb_drm_private *mxsfb = drm_pipe_to_mxsfb_drm_private(pipe);
+ drm_panel_prepare(mxsfb->panel);
mxsfb_crtc_enable(mxsfb);
+ drm_panel_enable(mxsfb->panel);
}
static void mxsfb_pipe_disable(struct drm_simple_display_pipe *pipe)
{
struct mxsfb_drm_private *mxsfb = drm_pipe_to_mxsfb_drm_private(pipe);
+ drm_panel_disable(mxsfb->panel);
mxsfb_crtc_disable(mxsfb);
+ drm_panel_unprepare(mxsfb->panel);
}
static void mxsfb_pipe_update(struct drm_simple_display_pipe *pipe,
int mxsfb_create_output(struct drm_device *drm)
{
+ struct mxsfb_drm_private *mxsfb = drm->dev_private;
struct device_node *ep_np = NULL;
struct of_endpoint ep;
int ret;
}
}
+ if (!mxsfb->panel)
+ return -EPROBE_DEFER;
+
return 0;
}
#define CTRL_DATA_SELECT (1 << 16)
#define CTRL_SET_BUS_WIDTH(x) (((x) & 0x3) << 10)
#define CTRL_GET_BUS_WIDTH(x) (((x) >> 10) & 0x3)
+#define CTRL_BUS_WIDTH_MASK (0x3 << 10)
#define CTRL_SET_WORD_LENGTH(x) (((x) & 0x3) << 8)
#define CTRL_GET_WORD_LENGTH(x) (((x) >> 8) & 0x3)
#define CTRL_MASTER (1 << 5)
#include <linux/jiffies.h>
#include <linux/module.h>
#include <linux/platform_device.h>
-#include <linux/sched.h>
+#include <linux/sched/signal.h>
#include <linux/slab.h>
#include <linux/workqueue.h>
#include <linux/of_device.h>
{
const struct drm_display_mode *mode = &crtc->crtc.state->adjusted_mode;
struct rcar_du_device *rcdu = crtc->group->dev;
+ struct vsp1_du_lif_config cfg = {
+ .width = mode->hdisplay,
+ .height = mode->vdisplay,
+ };
struct rcar_du_plane_state state = {
.state = {
.crtc = &crtc->crtc,
*/
crtc->group->need_restart = true;
- vsp1_du_setup_lif(crtc->vsp->vsp, mode->hdisplay, mode->vdisplay);
+ vsp1_du_setup_lif(crtc->vsp->vsp, &cfg);
}
void rcar_du_vsp_disable(struct rcar_du_crtc *crtc)
{
- vsp1_du_setup_lif(crtc->vsp->vsp, 0, 0);
+ vsp1_du_setup_lif(crtc->vsp->vsp, NULL);
}
void rcar_du_vsp_atomic_begin(struct rcar_du_crtc *crtc)
#include <linux/atomic.h>
#include <linux/errno.h>
#include <linux/wait.h>
-#include <linux/sched.h>
+#include <linux/sched/signal.h>
#include <linux/module.h>
#define TTM_WRITE_LOCK_PENDING (1 << 0)
#include <linux/pm_runtime.h>
#include <linux/device.h>
#include <linux/io.h>
+#include <linux/sched/signal.h>
#include "uapi/drm/vc4_drm.h"
#include "vc4_drv.h"
if (!obj->filp)
return -ENODEV;
- ret = obj->filp->f_op->mmap(obj->filp, vma);
+ ret = call_mmap(obj->filp, vma);
if (ret)
return ret;
#endif
ret = vgdev->vdev->config->find_vqs(vgdev->vdev, 2, vqs,
- callbacks, names);
+ callbacks, names, NULL);
if (ret) {
DRM_ERROR("failed to find virt queues\n");
goto err_vqs;
#include <linux/errno.h>
#include <linux/init.h>
#include <linux/list.h>
-#include <linux/sched.h>
+#include <linux/sched/signal.h>
#include <linux/wait.h>
#include <linux/spinlock.h>
#include <linux/poll.h>
#include <linux/debugfs.h>
#include <linux/seq_file.h>
-#include <linux/sched.h>
+#include <linux/sched/signal.h>
#include <linux/export.h>
#include <linux/slab.h>
#include <linux/uaccess.h>
#include <linux/cdev.h>
#include <linux/poll.h>
-#include <linux/sched.h>
+#include <linux/sched/signal.h>
#include <linux/hid-roccat.h>
#include <linux/module.h>
#include <linux/slab.h>
#include <linux/hid.h>
#include <linux/mutex.h>
-#include <linux/sched.h>
+#include <linux/sched/signal.h>
#include <linux/string.h>
#include <linux/hidraw.h>
#include <linux/poll.h>
#include <linux/slab.h>
+#include <linux/sched/signal.h>
#include <linux/module.h>
#include <linux/init.h>
#include <linux/input.h>
#include <linux/slab.h>
#include <linux/fs.h>
#include <linux/poll.h>
-#include <linux/sched.h>
+#include <linux/sched/signal.h>
#include <linux/ioctl.h>
#include <linux/uaccess.h>
#include <linux/pm_qos.h>
/*
* In case a device driver's probe() fails (e.g.,
* util_probe() -> vmbus_open() returns -ENOMEM) and the device is
- * rescinded later (e.g., we dynamically disble an Integrated Service
+ * rescinded later (e.g., we dynamically disable an Integrated Service
* in Hyper-V Manager), the driver's remove() invokes vmbus_close():
* here we should skip most of the below cleanup work.
*/
#include <linux/kernel_stat.h>
#include <linux/clockchips.h>
#include <linux/cpu.h>
+#include <linux/sched/task_stack.h>
+
#include <asm/hyperv.h>
#include <asm/hypervisor.h>
#include <asm/mshyperv.h>
u8 *tmp_buf;
int len = 0;
int xfersz = brcmstb_i2c_get_xfersz(dev);
+ u32 cond, cond_per_msg;
if (dev->is_suspended)
return -EBUSY;
pmsg->buf ? pmsg->buf[0] : '0', pmsg->len);
if (i < (num - 1) && (msgs[i + 1].flags & I2C_M_NOSTART))
- brcmstb_set_i2c_start_stop(dev, ~(COND_START_STOP));
+ cond = ~COND_START_STOP;
else
- brcmstb_set_i2c_start_stop(dev,
- COND_RESTART | COND_NOSTOP);
+ cond = COND_RESTART | COND_NOSTOP;
+
+ brcmstb_set_i2c_start_stop(dev, cond);
/* Send slave address */
if (!(pmsg->flags & I2C_M_NOSTART)) {
}
}
+ cond_per_msg = cond;
+
/* Perform data transfer */
while (len) {
bytes_to_xfer = min(len, xfersz);
- if (len <= xfersz && i == (num - 1))
- brcmstb_set_i2c_start_stop(dev,
- ~(COND_START_STOP));
+ if (len <= xfersz) {
+ if (i == (num - 1))
+ cond_per_msg = cond_per_msg &
+ ~(COND_RESTART | COND_NOSTOP);
+ else
+ cond_per_msg = cond;
+ } else {
+ cond_per_msg = (cond_per_msg & ~COND_RESTART) |
+ COND_NOSTOP;
+ }
+
+ brcmstb_set_i2c_start_stop(dev, cond_per_msg);
rc = brcmstb_i2c_xfer_bsc_data(dev, tmp_buf,
bytes_to_xfer, pmsg);
len -= bytes_to_xfer;
tmp_buf += bytes_to_xfer;
+
+ cond_per_msg = COND_NOSTART | COND_NOSTOP;
}
}
void __iomem *base;
struct completion cmd_complete;
struct clk *clk;
+ struct reset_control *rst;
u32 (*get_clk_rate_khz) (struct dw_i2c_dev *dev);
struct dw_pci_controller *controller;
int cmd_err;
#include <linux/pm_runtime.h>
#include <linux/property.h>
#include <linux/io.h>
+#include <linux/reset.h>
#include <linux/slab.h>
#include <linux/acpi.h>
#include <linux/platform_data/i2c-designware.h>
dev->irq = irq;
platform_set_drvdata(pdev, dev);
+ dev->rst = devm_reset_control_get_optional_exclusive(&pdev->dev, NULL);
+ if (IS_ERR(dev->rst)) {
+ if (PTR_ERR(dev->rst) == -EPROBE_DEFER)
+ return -EPROBE_DEFER;
+ } else {
+ reset_control_deassert(dev->rst);
+ }
+
if (pdata) {
dev->clk_freq = pdata->i2c_scl_freq;
} else {
&& dev->clk_freq != 1000000 && dev->clk_freq != 3400000) {
dev_err(&pdev->dev,
"Only 100kHz, 400kHz, 1MHz and 3.4MHz supported");
- return -EINVAL;
+ r = -EINVAL;
+ goto exit_reset;
}
r = i2c_dw_eval_lock_support(dev);
if (r)
- return r;
+ goto exit_reset;
dev->functionality = I2C_FUNC_10BIT_ADDR | DW_IC_DEFAULT_FUNCTIONALITY;
}
r = i2c_dw_probe(dev);
- if (r && !dev->pm_runtime_disabled)
- pm_runtime_disable(&pdev->dev);
+ if (r)
+ goto exit_probe;
return r;
+
+exit_probe:
+ if (!dev->pm_runtime_disabled)
+ pm_runtime_disable(&pdev->dev);
+exit_reset:
+ if (!IS_ERR_OR_NULL(dev->rst))
+ reset_control_assert(dev->rst);
+ return r;
}
static int dw_i2c_plat_remove(struct platform_device *pdev)
pm_runtime_put_sync(&pdev->dev);
if (!dev->pm_runtime_disabled)
pm_runtime_disable(&pdev->dev);
+ if (!IS_ERR_OR_NULL(dev->rst))
+ reset_control_assert(dev->rst);
return 0;
}
int_status = readl(i2c->regs + HSI2C_INT_STATUS);
writel(int_status, i2c->regs + HSI2C_INT_STATUS);
- trans_status = readl(i2c->regs + HSI2C_TRANS_STATUS);
/* handle interrupt related to the transfer status */
if (i2c->variant->hw == HSI2C_EXYNOS7) {
goto stop;
}
+ trans_status = readl(i2c->regs + HSI2C_TRANS_STATUS);
if ((trans_status & HSI2C_MASTER_ST_MASK) == HSI2C_MASTER_ST_LOSE) {
i2c->state = -EAGAIN;
goto stop;
}
} else if (int_status & HSI2C_INT_I2C) {
+ trans_status = readl(i2c->regs + HSI2C_TRANS_STATUS);
if (trans_status & HSI2C_NO_DEV_ACK) {
dev_dbg(i2c->dev, "No ACK from device\n");
i2c->state = -ENXIO;
#include <linux/delay.h>
#include <linux/slab.h>
#include <linux/interrupt.h>
+#include <linux/sched/signal.h>
+
#include <asm/irq.h>
#include <linux/io.h>
#include <linux/i2c.h>
wdata1 |= *buf++ << ((i - 4) * 8);
writel(wdata0, i2c->regs + REG_TOK_WDATA0);
- writel(wdata0, i2c->regs + REG_TOK_WDATA1);
+ writel(wdata1, i2c->regs + REG_TOK_WDATA1);
dev_dbg(i2c->dev, "%s: data %08x %08x len %d\n", __func__,
wdata0, wdata1, len);
#include <linux/kernel.h>
#include <linux/module.h>
-#include <linux/sched.h>
+#include <linux/sched/signal.h>
#include <linux/of_address.h>
#include <linux/of_irq.h>
#include <linux/of_platform.h>
.max_comb_2nd_msg_len = 31,
};
-static const struct i2c_adapter_quirks mt8173_i2c_quirks = {
- .max_num_msgs = 65535,
- .max_write_len = 65535,
- .max_read_len = 65535,
- .max_comb_1st_msg_len = 65535,
- .max_comb_2nd_msg_len = 65535,
-};
-
static const struct mtk_i2c_compatible mt6577_compat = {
.quirks = &mt6577_i2c_quirks,
.pmic_i2c = 0,
};
static const struct mtk_i2c_compatible mt8173_compat = {
- .quirks = &mt8173_i2c_quirks,
.pmic_i2c = 0,
.dcm = 1,
.auto_restart = 1,
}
if (riic->is_last || riic->err) {
- riic_clear_set_bit(riic, 0, ICIER_SPIE, RIIC_ICIER);
+ riic_clear_set_bit(riic, ICIER_TEIE, ICIER_SPIE, RIIC_ICIER);
writeb(ICCR2_SP, riic->base + RIIC_ICCR2);
+ } else {
+ /* Transfer is complete, but do not send STOP */
+ riic_clear_set_bit(riic, ICIER_TEIE, 0, RIIC_ICIER);
+ complete(&riic->msg_done);
}
return IRQ_HANDLED;
while (muxc->num_adapters) {
struct i2c_adapter *adap = muxc->adapter[--muxc->num_adapters];
struct i2c_mux_priv *priv = adap->algo_data;
+ struct device_node *np = adap->dev.of_node;
muxc->adapter[muxc->num_adapters] = NULL;
sysfs_remove_link(&priv->adap.dev.kobj, "mux_device");
i2c_del_adapter(adap);
+ of_node_put(np);
kfree(priv);
}
}
#include <linux/module.h>
#include <linux/types.h>
#include <linux/kernel.h>
+#include <linux/sched/task_stack.h>
#include <linux/delay.h>
#include <linux/timer.h>
#include <linux/seq_file.h>
#include <linux/delay.h>
#include <linux/hdreg.h>
#include <linux/ide.h>
+#include <linux/nmi.h>
#include <linux/scatterlist.h>
#include <linux/uaccess.h>
*/
static struct cpuidle_state nehalem_cstates[] = {
{
- .name = "C1-NHM",
+ .name = "C1",
.desc = "MWAIT 0x00",
.flags = MWAIT2flg(0x00),
.exit_latency = 3,
.enter = &intel_idle,
.enter_freeze = intel_idle_freeze, },
{
- .name = "C1E-NHM",
+ .name = "C1E",
.desc = "MWAIT 0x01",
.flags = MWAIT2flg(0x01),
.exit_latency = 10,
.enter = &intel_idle,
.enter_freeze = intel_idle_freeze, },
{
- .name = "C3-NHM",
+ .name = "C3",
.desc = "MWAIT 0x10",
.flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED,
.exit_latency = 20,
.enter = &intel_idle,
.enter_freeze = intel_idle_freeze, },
{
- .name = "C6-NHM",
+ .name = "C6",
.desc = "MWAIT 0x20",
.flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
.exit_latency = 200,
static struct cpuidle_state snb_cstates[] = {
{
- .name = "C1-SNB",
+ .name = "C1",
.desc = "MWAIT 0x00",
.flags = MWAIT2flg(0x00),
.exit_latency = 2,
.enter = &intel_idle,
.enter_freeze = intel_idle_freeze, },
{
- .name = "C1E-SNB",
+ .name = "C1E",
.desc = "MWAIT 0x01",
.flags = MWAIT2flg(0x01),
.exit_latency = 10,
.enter = &intel_idle,
.enter_freeze = intel_idle_freeze, },
{
- .name = "C3-SNB",
+ .name = "C3",
.desc = "MWAIT 0x10",
.flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED,
.exit_latency = 80,
.enter = &intel_idle,
.enter_freeze = intel_idle_freeze, },
{
- .name = "C6-SNB",
+ .name = "C6",
.desc = "MWAIT 0x20",
.flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
.exit_latency = 104,
.enter = &intel_idle,
.enter_freeze = intel_idle_freeze, },
{
- .name = "C7-SNB",
+ .name = "C7",
.desc = "MWAIT 0x30",
.flags = MWAIT2flg(0x30) | CPUIDLE_FLAG_TLB_FLUSHED,
.exit_latency = 109,
static struct cpuidle_state byt_cstates[] = {
{
- .name = "C1-BYT",
+ .name = "C1",
.desc = "MWAIT 0x00",
.flags = MWAIT2flg(0x00),
.exit_latency = 1,
.enter = &intel_idle,
.enter_freeze = intel_idle_freeze, },
{
- .name = "C6N-BYT",
+ .name = "C6N",
.desc = "MWAIT 0x58",
.flags = MWAIT2flg(0x58) | CPUIDLE_FLAG_TLB_FLUSHED,
.exit_latency = 300,
.enter = &intel_idle,
.enter_freeze = intel_idle_freeze, },
{
- .name = "C6S-BYT",
+ .name = "C6S",
.desc = "MWAIT 0x52",
.flags = MWAIT2flg(0x52) | CPUIDLE_FLAG_TLB_FLUSHED,
.exit_latency = 500,
.enter = &intel_idle,
.enter_freeze = intel_idle_freeze, },
{
- .name = "C7-BYT",
+ .name = "C7",
.desc = "MWAIT 0x60",
.flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED,
.exit_latency = 1200,
.enter = &intel_idle,
.enter_freeze = intel_idle_freeze, },
{
- .name = "C7S-BYT",
+ .name = "C7S",
.desc = "MWAIT 0x64",
.flags = MWAIT2flg(0x64) | CPUIDLE_FLAG_TLB_FLUSHED,
.exit_latency = 10000,
static struct cpuidle_state cht_cstates[] = {
{
- .name = "C1-CHT",
+ .name = "C1",
.desc = "MWAIT 0x00",
.flags = MWAIT2flg(0x00),
.exit_latency = 1,
.enter = &intel_idle,
.enter_freeze = intel_idle_freeze, },
{
- .name = "C6N-CHT",
+ .name = "C6N",
.desc = "MWAIT 0x58",
.flags = MWAIT2flg(0x58) | CPUIDLE_FLAG_TLB_FLUSHED,
.exit_latency = 80,
.enter = &intel_idle,
.enter_freeze = intel_idle_freeze, },
{
- .name = "C6S-CHT",
+ .name = "C6S",
.desc = "MWAIT 0x52",
.flags = MWAIT2flg(0x52) | CPUIDLE_FLAG_TLB_FLUSHED,
.exit_latency = 200,
.enter = &intel_idle,
.enter_freeze = intel_idle_freeze, },
{
- .name = "C7-CHT",
+ .name = "C7",
.desc = "MWAIT 0x60",
.flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED,
.exit_latency = 1200,
.enter = &intel_idle,
.enter_freeze = intel_idle_freeze, },
{
- .name = "C7S-CHT",
+ .name = "C7S",
.desc = "MWAIT 0x64",
.flags = MWAIT2flg(0x64) | CPUIDLE_FLAG_TLB_FLUSHED,
.exit_latency = 10000,
static struct cpuidle_state ivb_cstates[] = {
{
- .name = "C1-IVB",
+ .name = "C1",
.desc = "MWAIT 0x00",
.flags = MWAIT2flg(0x00),
.exit_latency = 1,
.enter = &intel_idle,
.enter_freeze = intel_idle_freeze, },
{
- .name = "C1E-IVB",
+ .name = "C1E",
.desc = "MWAIT 0x01",
.flags = MWAIT2flg(0x01),
.exit_latency = 10,
.enter = &intel_idle,
.enter_freeze = intel_idle_freeze, },
{
- .name = "C3-IVB",
+ .name = "C3",
.desc = "MWAIT 0x10",
.flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED,
.exit_latency = 59,
.enter = &intel_idle,
.enter_freeze = intel_idle_freeze, },
{
- .name = "C6-IVB",
+ .name = "C6",
.desc = "MWAIT 0x20",
.flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
.exit_latency = 80,
.enter = &intel_idle,
.enter_freeze = intel_idle_freeze, },
{
- .name = "C7-IVB",
+ .name = "C7",
.desc = "MWAIT 0x30",
.flags = MWAIT2flg(0x30) | CPUIDLE_FLAG_TLB_FLUSHED,
.exit_latency = 87,
static struct cpuidle_state ivt_cstates[] = {
{
- .name = "C1-IVT",
+ .name = "C1",
.desc = "MWAIT 0x00",
.flags = MWAIT2flg(0x00),
.exit_latency = 1,
.enter = &intel_idle,
.enter_freeze = intel_idle_freeze, },
{
- .name = "C1E-IVT",
+ .name = "C1E",
.desc = "MWAIT 0x01",
.flags = MWAIT2flg(0x01),
.exit_latency = 10,
.enter = &intel_idle,
.enter_freeze = intel_idle_freeze, },
{
- .name = "C3-IVT",
+ .name = "C3",
.desc = "MWAIT 0x10",
.flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED,
.exit_latency = 59,
.enter = &intel_idle,
.enter_freeze = intel_idle_freeze, },
{
- .name = "C6-IVT",
+ .name = "C6",
.desc = "MWAIT 0x20",
.flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
.exit_latency = 82,
static struct cpuidle_state ivt_cstates_4s[] = {
{
- .name = "C1-IVT-4S",
+ .name = "C1",
.desc = "MWAIT 0x00",
.flags = MWAIT2flg(0x00),
.exit_latency = 1,
.enter = &intel_idle,
.enter_freeze = intel_idle_freeze, },
{
- .name = "C1E-IVT-4S",
+ .name = "C1E",
.desc = "MWAIT 0x01",
.flags = MWAIT2flg(0x01),
.exit_latency = 10,
.enter = &intel_idle,
.enter_freeze = intel_idle_freeze, },
{
- .name = "C3-IVT-4S",
+ .name = "C3",
.desc = "MWAIT 0x10",
.flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED,
.exit_latency = 59,
.enter = &intel_idle,
.enter_freeze = intel_idle_freeze, },
{
- .name = "C6-IVT-4S",
+ .name = "C6",
.desc = "MWAIT 0x20",
.flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
.exit_latency = 84,
static struct cpuidle_state ivt_cstates_8s[] = {
{
- .name = "C1-IVT-8S",
+ .name = "C1",
.desc = "MWAIT 0x00",
.flags = MWAIT2flg(0x00),
.exit_latency = 1,
.enter = &intel_idle,
.enter_freeze = intel_idle_freeze, },
{
- .name = "C1E-IVT-8S",
+ .name = "C1E",
.desc = "MWAIT 0x01",
.flags = MWAIT2flg(0x01),
.exit_latency = 10,
.enter = &intel_idle,
.enter_freeze = intel_idle_freeze, },
{
- .name = "C3-IVT-8S",
+ .name = "C3",
.desc = "MWAIT 0x10",
.flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED,
.exit_latency = 59,
.enter = &intel_idle,
.enter_freeze = intel_idle_freeze, },
{
- .name = "C6-IVT-8S",
+ .name = "C6",
.desc = "MWAIT 0x20",
.flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
.exit_latency = 88,
static struct cpuidle_state hsw_cstates[] = {
{
- .name = "C1-HSW",
+ .name = "C1",
.desc = "MWAIT 0x00",
.flags = MWAIT2flg(0x00),
.exit_latency = 2,
.enter = &intel_idle,
.enter_freeze = intel_idle_freeze, },
{
- .name = "C1E-HSW",
+ .name = "C1E",
.desc = "MWAIT 0x01",
.flags = MWAIT2flg(0x01),
.exit_latency = 10,
.enter = &intel_idle,
.enter_freeze = intel_idle_freeze, },
{
- .name = "C3-HSW",
+ .name = "C3",
.desc = "MWAIT 0x10",
.flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED,
.exit_latency = 33,
.enter = &intel_idle,
.enter_freeze = intel_idle_freeze, },
{
- .name = "C6-HSW",
+ .name = "C6",
.desc = "MWAIT 0x20",
.flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
.exit_latency = 133,
.enter = &intel_idle,
.enter_freeze = intel_idle_freeze, },
{
- .name = "C7s-HSW",
+ .name = "C7s",
.desc = "MWAIT 0x32",
.flags = MWAIT2flg(0x32) | CPUIDLE_FLAG_TLB_FLUSHED,
.exit_latency = 166,
.enter = &intel_idle,
.enter_freeze = intel_idle_freeze, },
{
- .name = "C8-HSW",
+ .name = "C8",
.desc = "MWAIT 0x40",
.flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED,
.exit_latency = 300,
.enter = &intel_idle,
.enter_freeze = intel_idle_freeze, },
{
- .name = "C9-HSW",
+ .name = "C9",
.desc = "MWAIT 0x50",
.flags = MWAIT2flg(0x50) | CPUIDLE_FLAG_TLB_FLUSHED,
.exit_latency = 600,
.enter = &intel_idle,
.enter_freeze = intel_idle_freeze, },
{
- .name = "C10-HSW",
+ .name = "C10",
.desc = "MWAIT 0x60",
.flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED,
.exit_latency = 2600,
};
static struct cpuidle_state bdw_cstates[] = {
{
- .name = "C1-BDW",
+ .name = "C1",
.desc = "MWAIT 0x00",
.flags = MWAIT2flg(0x00),
.exit_latency = 2,
.enter = &intel_idle,
.enter_freeze = intel_idle_freeze, },
{
- .name = "C1E-BDW",
+ .name = "C1E",
.desc = "MWAIT 0x01",
.flags = MWAIT2flg(0x01),
.exit_latency = 10,
.enter = &intel_idle,
.enter_freeze = intel_idle_freeze, },
{
- .name = "C3-BDW",
+ .name = "C3",
.desc = "MWAIT 0x10",
.flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED,
.exit_latency = 40,
.enter = &intel_idle,
.enter_freeze = intel_idle_freeze, },
{
- .name = "C6-BDW",
+ .name = "C6",
.desc = "MWAIT 0x20",
.flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
.exit_latency = 133,
.enter = &intel_idle,
.enter_freeze = intel_idle_freeze, },
{
- .name = "C7s-BDW",
+ .name = "C7s",
.desc = "MWAIT 0x32",
.flags = MWAIT2flg(0x32) | CPUIDLE_FLAG_TLB_FLUSHED,
.exit_latency = 166,
.enter = &intel_idle,
.enter_freeze = intel_idle_freeze, },
{
- .name = "C8-BDW",
+ .name = "C8",
.desc = "MWAIT 0x40",
.flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED,
.exit_latency = 300,
.enter = &intel_idle,
.enter_freeze = intel_idle_freeze, },
{
- .name = "C9-BDW",
+ .name = "C9",
.desc = "MWAIT 0x50",
.flags = MWAIT2flg(0x50) | CPUIDLE_FLAG_TLB_FLUSHED,
.exit_latency = 600,
.enter = &intel_idle,
.enter_freeze = intel_idle_freeze, },
{
- .name = "C10-BDW",
+ .name = "C10",
.desc = "MWAIT 0x60",
.flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED,
.exit_latency = 2600,
static struct cpuidle_state skl_cstates[] = {
{
- .name = "C1-SKL",
+ .name = "C1",
.desc = "MWAIT 0x00",
.flags = MWAIT2flg(0x00),
.exit_latency = 2,
.enter = &intel_idle,
.enter_freeze = intel_idle_freeze, },
{
- .name = "C1E-SKL",
+ .name = "C1E",
.desc = "MWAIT 0x01",
.flags = MWAIT2flg(0x01),
.exit_latency = 10,
.enter = &intel_idle,
.enter_freeze = intel_idle_freeze, },
{
- .name = "C3-SKL",
+ .name = "C3",
.desc = "MWAIT 0x10",
.flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED,
.exit_latency = 70,
.enter = &intel_idle,
.enter_freeze = intel_idle_freeze, },
{
- .name = "C6-SKL",
+ .name = "C6",
.desc = "MWAIT 0x20",
.flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
.exit_latency = 85,
.enter = &intel_idle,
.enter_freeze = intel_idle_freeze, },
{
- .name = "C7s-SKL",
+ .name = "C7s",
.desc = "MWAIT 0x33",
.flags = MWAIT2flg(0x33) | CPUIDLE_FLAG_TLB_FLUSHED,
.exit_latency = 124,
.enter = &intel_idle,
.enter_freeze = intel_idle_freeze, },
{
- .name = "C8-SKL",
+ .name = "C8",
.desc = "MWAIT 0x40",
.flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED,
.exit_latency = 200,
.enter = &intel_idle,
.enter_freeze = intel_idle_freeze, },
{
- .name = "C9-SKL",
+ .name = "C9",
.desc = "MWAIT 0x50",
.flags = MWAIT2flg(0x50) | CPUIDLE_FLAG_TLB_FLUSHED,
.exit_latency = 480,
.enter = &intel_idle,
.enter_freeze = intel_idle_freeze, },
{
- .name = "C10-SKL",
+ .name = "C10",
.desc = "MWAIT 0x60",
.flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED,
.exit_latency = 890,
static struct cpuidle_state skx_cstates[] = {
{
- .name = "C1-SKX",
+ .name = "C1",
.desc = "MWAIT 0x00",
.flags = MWAIT2flg(0x00),
.exit_latency = 2,
.enter = &intel_idle,
.enter_freeze = intel_idle_freeze, },
{
- .name = "C1E-SKX",
+ .name = "C1E",
.desc = "MWAIT 0x01",
.flags = MWAIT2flg(0x01),
.exit_latency = 10,
.enter = &intel_idle,
.enter_freeze = intel_idle_freeze, },
{
- .name = "C6-SKX",
+ .name = "C6",
.desc = "MWAIT 0x20",
.flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
.exit_latency = 133,
static struct cpuidle_state atom_cstates[] = {
{
- .name = "C1E-ATM",
+ .name = "C1E",
.desc = "MWAIT 0x00",
.flags = MWAIT2flg(0x00),
.exit_latency = 10,
.enter = &intel_idle,
.enter_freeze = intel_idle_freeze, },
{
- .name = "C2-ATM",
+ .name = "C2",
.desc = "MWAIT 0x10",
.flags = MWAIT2flg(0x10),
.exit_latency = 20,
.enter = &intel_idle,
.enter_freeze = intel_idle_freeze, },
{
- .name = "C4-ATM",
+ .name = "C4",
.desc = "MWAIT 0x30",
.flags = MWAIT2flg(0x30) | CPUIDLE_FLAG_TLB_FLUSHED,
.exit_latency = 100,
.enter = &intel_idle,
.enter_freeze = intel_idle_freeze, },
{
- .name = "C6-ATM",
+ .name = "C6",
.desc = "MWAIT 0x52",
.flags = MWAIT2flg(0x52) | CPUIDLE_FLAG_TLB_FLUSHED,
.exit_latency = 140,
};
static struct cpuidle_state tangier_cstates[] = {
{
- .name = "C1-TNG",
+ .name = "C1",
.desc = "MWAIT 0x00",
.flags = MWAIT2flg(0x00),
.exit_latency = 1,
.enter = &intel_idle,
.enter_freeze = intel_idle_freeze, },
{
- .name = "C4-TNG",
+ .name = "C4",
.desc = "MWAIT 0x30",
.flags = MWAIT2flg(0x30) | CPUIDLE_FLAG_TLB_FLUSHED,
.exit_latency = 100,
.enter = &intel_idle,
.enter_freeze = intel_idle_freeze, },
{
- .name = "C6-TNG",
+ .name = "C6",
.desc = "MWAIT 0x52",
.flags = MWAIT2flg(0x52) | CPUIDLE_FLAG_TLB_FLUSHED,
.exit_latency = 140,
.enter = &intel_idle,
.enter_freeze = intel_idle_freeze, },
{
- .name = "C7-TNG",
+ .name = "C7",
.desc = "MWAIT 0x60",
.flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED,
.exit_latency = 1200,
.enter = &intel_idle,
.enter_freeze = intel_idle_freeze, },
{
- .name = "C9-TNG",
+ .name = "C9",
.desc = "MWAIT 0x64",
.flags = MWAIT2flg(0x64) | CPUIDLE_FLAG_TLB_FLUSHED,
.exit_latency = 10000,
};
static struct cpuidle_state avn_cstates[] = {
{
- .name = "C1-AVN",
+ .name = "C1",
.desc = "MWAIT 0x00",
.flags = MWAIT2flg(0x00),
.exit_latency = 2,
.enter = &intel_idle,
.enter_freeze = intel_idle_freeze, },
{
- .name = "C6-AVN",
+ .name = "C6",
.desc = "MWAIT 0x51",
.flags = MWAIT2flg(0x51) | CPUIDLE_FLAG_TLB_FLUSHED,
.exit_latency = 15,
};
static struct cpuidle_state knl_cstates[] = {
{
- .name = "C1-KNL",
+ .name = "C1",
.desc = "MWAIT 0x00",
.flags = MWAIT2flg(0x00),
.exit_latency = 1,
.enter = &intel_idle,
.enter_freeze = intel_idle_freeze },
{
- .name = "C6-KNL",
+ .name = "C6",
.desc = "MWAIT 0x10",
.flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED,
.exit_latency = 120,
static struct cpuidle_state bxt_cstates[] = {
{
- .name = "C1-BXT",
+ .name = "C1",
.desc = "MWAIT 0x00",
.flags = MWAIT2flg(0x00),
.exit_latency = 2,
.enter = &intel_idle,
.enter_freeze = intel_idle_freeze, },
{
- .name = "C1E-BXT",
+ .name = "C1E",
.desc = "MWAIT 0x01",
.flags = MWAIT2flg(0x01),
.exit_latency = 10,
.enter = &intel_idle,
.enter_freeze = intel_idle_freeze, },
{
- .name = "C6-BXT",
+ .name = "C6",
.desc = "MWAIT 0x20",
.flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
.exit_latency = 133,
.enter = &intel_idle,
.enter_freeze = intel_idle_freeze, },
{
- .name = "C7s-BXT",
+ .name = "C7s",
.desc = "MWAIT 0x31",
.flags = MWAIT2flg(0x31) | CPUIDLE_FLAG_TLB_FLUSHED,
.exit_latency = 155,
.enter = &intel_idle,
.enter_freeze = intel_idle_freeze, },
{
- .name = "C8-BXT",
+ .name = "C8",
.desc = "MWAIT 0x40",
.flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED,
.exit_latency = 1000,
.enter = &intel_idle,
.enter_freeze = intel_idle_freeze, },
{
- .name = "C9-BXT",
+ .name = "C9",
.desc = "MWAIT 0x50",
.flags = MWAIT2flg(0x50) | CPUIDLE_FLAG_TLB_FLUSHED,
.exit_latency = 2000,
.enter = &intel_idle,
.enter_freeze = intel_idle_freeze, },
{
- .name = "C10-BXT",
+ .name = "C10",
.desc = "MWAIT 0x60",
.flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED,
.exit_latency = 10000,
static struct cpuidle_state dnv_cstates[] = {
{
- .name = "C1-DNV",
+ .name = "C1",
.desc = "MWAIT 0x00",
.flags = MWAIT2flg(0x00),
.exit_latency = 2,
.enter = &intel_idle,
.enter_freeze = intel_idle_freeze, },
{
- .name = "C1E-DNV",
+ .name = "C1E",
.desc = "MWAIT 0x01",
.flags = MWAIT2flg(0x01),
.exit_latency = 10,
.enter = &intel_idle,
.enter_freeze = intel_idle_freeze, },
{
- .name = "C6-DNV",
+ .name = "C6",
.desc = "MWAIT 0x20",
.flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
.exit_latency = 50,
{
unsigned long long msr_bits;
- rdmsrl(MSR_NHM_SNB_PKG_CST_CFG_CTL, msr_bits);
+ rdmsrl(MSR_PKG_CST_CONFIG_CONTROL, msr_bits);
msr_bits &= ~(icpu->auto_demotion_disable_flags);
- wrmsrl(MSR_NHM_SNB_PKG_CST_CFG_CTL, msr_bits);
+ wrmsrl(MSR_PKG_CST_CONFIG_CONTROL, msr_bits);
}
static void c1e_promotion_disable(void)
{
if ((mwait_substates & (0xF << 28)) == 0)
return;
- rdmsrl(MSR_NHM_SNB_PKG_CST_CFG_CTL, msr);
+ rdmsrl(MSR_PKG_CST_CONFIG_CONTROL, msr);
/* PC10 is not enabled in PKG C-state limit */
if ((msr & 0xF) != 8)
struct device_node *child;
struct regulator *vref;
unsigned int reg;
- unsigned int adcmode, childmode;
+ unsigned int adcmode = -1, childmode;
unsigned int sample_width;
unsigned int num_channels;
int ret, first = 1;
channels = rcar_gyroadc_iio_channels_3;
num_channels = ARRAY_SIZE(rcar_gyroadc_iio_channels_3);
break;
+ default:
+ return -EINVAL;
}
/*
ret = xadc->ops->setup(pdev, indio_dev, irq);
if (ret)
- goto err_free_samplerate_trigger;
+ goto err_clk_disable_unprepare;
ret = request_irq(irq, xadc->ops->interrupt_handler, 0,
dev_name(&pdev->dev), indio_dev);
err_free_irq:
free_irq(irq, indio_dev);
+err_clk_disable_unprepare:
+ clk_disable_unprepare(xadc->clk);
err_free_samplerate_trigger:
if (xadc->ops->flags & XADC_FLAGS_BUFFERED)
iio_trigger_free(xadc->samplerate_trigger);
err_triggered_buffer_cleanup:
if (xadc->ops->flags & XADC_FLAGS_BUFFERED)
iio_triggered_buffer_cleanup(indio_dev);
-err_clk_disable_unprepare:
- clk_disable_unprepare(xadc->clk);
err_device_free:
kfree(indio_dev->channels);
return IIO_VAL_INT;
}
- flags = inb(base_offset);
+ flags = inb(base_offset + 1);
borrow = flags & BIT(0);
carry = !!(flags & BIT(1));
#include <linux/cdev.h>
#include <linux/slab.h>
#include <linux/poll.h>
-#include <linux/sched.h>
+#include <linux/sched/signal.h>
#include <linux/iio/iio.h>
#include "iio_core.h"
#include <linux/mm.h>
#include <linux/dma-mapping.h>
-#include <linux/sched.h>
+#include <linux/sched/signal.h>
+#include <linux/sched/mm.h>
#include <linux/export.h>
#include <linux/hugetlb.h>
#include <linux/slab.h>
#include <linux/types.h>
#include <linux/sched.h>
+#include <linux/sched/mm.h>
+#include <linux/sched/task.h>
#include <linux/pid.h>
#include <linux/slab.h>
#include <linux/export.h>
#include <linux/delay.h>
#include <linux/errno.h>
#include <linux/list.h>
-#include <linux/sched.h>
+#include <linux/sched/mm.h>
#include <linux/spinlock.h>
#include <linux/ethtool.h>
#include <linux/rtnetlink.h>
#include <linux/idr.h>
#include <linux/completion.h>
#include <linux/netdevice.h>
-#include <linux/sched.h>
+#include <linux/sched/mm.h>
#include <linux/pci.h>
#include <linux/dma-mapping.h>
#include <linux/inet.h>
struct hfi1_affinity_node *entry;
cpumask_var_t diff, hw_thread_mask, available_mask, intrs_mask;
const struct cpumask *node_mask,
- *proc_mask = tsk_cpus_allowed(current);
+ *proc_mask = ¤t->cpus_allowed;
struct hfi1_affinity_node_list *affinity = &node_affinity;
struct cpu_mask_set *set = &affinity->proc;
#include <linux/cdev.h>
#include <linux/vmalloc.h>
#include <linux/io.h>
+#include <linux/sched/mm.h>
#include <rdma/ib.h>
{
struct sdma_rht_node *rht_node;
struct sdma_engine *sde = NULL;
- const struct cpumask *current_mask = tsk_cpus_allowed(current);
+ const struct cpumask *current_mask = ¤t->cpus_allowed;
unsigned long cpu_id;
/*
*/
#include <linux/mm.h>
-#include <linux/sched.h>
+#include <linux/sched/signal.h>
#include <linux/device.h>
#include <linux/module.h>
#include <linux/inetdevice.h>
#include <linux/rtnetlink.h>
#include <linux/if_vlan.h>
+#include <linux/sched/mm.h>
+#include <linux/sched/task.h>
+
#include <net/ipv6.h>
#include <net/addrconf.h>
#include <net/devlink.h>
#include <asm/pat.h>
#endif
#include <linux/sched.h>
+#include <linux/sched/mm.h>
+#include <linux/sched/task.h>
#include <linux/delay.h>
#include <rdma/ib_user_verbs.h>
#include <rdma/ib_addr.h>
*/
#include <linux/mm.h>
+#include <linux/sched/signal.h>
#include <linux/device.h>
#include "qib.h"
#include <linux/mm.h>
#include <linux/dma-mapping.h>
-#include <linux/sched.h>
+#include <linux/sched/signal.h>
+#include <linux/sched/mm.h>
#include <linux/hugetlb.h>
#include <linux/iommu.h>
#include <linux/workqueue.h>
#include <linux/slab.h>
#include <linux/vmalloc.h>
#include <linux/moduleparam.h>
+#include <linux/sched/signal.h>
#include "ipoib.h"
*/
#include <linux/module.h>
+#include <linux/sched/signal.h>
#include <linux/init.h>
#include <linux/seq_file.h>
#include <linux/profile.h>
#include <linux/module.h>
#include <linux/sched.h>
+#include <linux/sched/mm.h>
#include <linux/iommu.h>
#include <linux/wait.h>
#include <linux/pci.h>
#include <linux/intel-iommu.h>
#include <linux/mmu_notifier.h>
#include <linux/sched.h>
+#include <linux/sched/mm.h>
#include <linux/slab.h>
#include <linux/intel-svm.h>
#include <linux/rculist.h>
static int __init crossbar_of_init(struct device_node *node)
{
- int i, size, max = 0, reserved = 0, entry;
+ u32 max = 0, entry, reg_size;
+ int i, size, reserved = 0;
const __be32 *irqsr;
int ret = -ENOMEM;
if (!cb->register_offsets)
goto err_irq_map;
- of_property_read_u32(node, "ti,reg-size", &size);
+ of_property_read_u32(node, "ti,reg-size", ®_size);
- switch (size) {
+ switch (reg_size) {
case 1:
cb->write = crossbar_writeb;
break;
continue;
cb->register_offsets[i] = reserved;
- reserved += size;
+ reserved += reg_size;
}
of_property_read_u32(node, "ti,irqs-safe-map", &cb->safe_map);
its->flags |= ITS_FLAGS_WORKAROUND_CAVIUM_23144;
}
+static void __maybe_unused its_enable_quirk_qdf2400_e0065(void *data)
+{
+ struct its_node *its = data;
+
+ /* On QDF2400, the size of the ITE is 16Bytes */
+ its->ite_size = 16;
+}
+
static const struct gic_quirk its_quirks[] = {
#ifdef CONFIG_CAVIUM_ERRATUM_22375
{
.mask = 0xffff0fff,
.init = its_enable_quirk_cavium_23144,
},
+#endif
+#ifdef CONFIG_QCOM_QDF2400_ERRATUM_0065
+ {
+ .desc = "ITS: QDF2400 erratum 0065",
+ .iidr = 0x00001070, /* QDF2400 ITS rev 1.x */
+ .mask = 0xffffffff,
+ .init = its_enable_quirk_qdf2400_e0065,
+ },
#endif
{
}
#include <linux/interrupt.h>
#include <linux/ioport.h>
#include <linux/proc_fs.h>
-#include <linux/sched.h>
+#include <linux/sched/signal.h>
#include <linux/seq_file.h>
#include <linux/skbuff.h>
#include <linux/workqueue.h>
}
}
} else {
- // Disble B channel interrupts
+ // Disable B channel interrupts
st5481_usb_device_ctrl_msg(adapter, FFMSK_B1+(bcs->channel * 2), 0, NULL, NULL);
// Disable B channel FIFOs
#include <linux/slab.h>
#include <linux/delay.h>
#include <linux/mutex.h>
+#include <linux/sched/signal.h>
#include "isdn_common.h"
#include "isdn_tty.h"
#ifdef CONFIG_ISDN_AUDIO
#include <linux/workqueue.h>
#include <linux/kthread.h>
#include <linux/slab.h>
+#include <linux/sched/signal.h>
+
#include <net/sock.h>
#include "core.h"
#include "l1oip.h"
#include <linux/mISDNif.h>
#include <linux/kthread.h>
#include <linux/sched.h>
+#include <linux/sched/cputime.h>
+#include <linux/signal.h>
+
#include "core.h"
static u_int *debug;
#include <linux/module.h>
#include <linux/mISDNif.h>
#include <linux/mutex.h>
+#include <linux/sched/signal.h>
+
#include "core.h"
static DEFINE_MUTEX(mISDN_mutex);
#include <linux/slab.h>
#include <linux/timer.h>
#include <linux/sched.h>
+#include <linux/sched/loadavg.h>
#include <linux/leds.h>
#include <linux/reboot.h>
#include <linux/suspend.h>
#include <linux/stddef.h>
#include <linux/io.h>
#include <linux/mm.h>
+#include <linux/sched/signal.h>
#include <linux/vmalloc.h>
#include <linux/cpu.h>
#include <linux/freezer.h>
#include <linux/miscdevice.h>
#include <linux/fs.h>
#include <linux/sched.h>
+#include <linux/sched/mm.h>
#include <linux/file.h>
#include <linux/slab.h>
#include <linux/export.h>
#include <linux/module.h>
#include <linux/fs.h>
#include <linux/mm.h>
-#include <linux/sched.h>
+#include <linux/sched/signal.h>
#include <linux/adb.h>
#include <linux/cuda.h>
#include <linux/pmu.h>
#include <linux/of_platform.h>
#include <linux/slab.h>
#include <linux/memblock.h>
+#include <linux/sched/signal.h>
#include <asm/byteorder.h>
#include <asm/io.h>
#include <linux/errno.h>
#include <linux/kernel.h>
#include <linux/delay.h>
-#include <linux/sched.h>
+#include <linux/sched/signal.h>
#include <linux/miscdevice.h>
#include <linux/blkdev.h>
#include <linux/pci.h>
#include <linux/poll.h>
#include <linux/slab.h>
#include <linux/uaccess.h>
+#include <linux/sched/signal.h>
#define MBOX_MAX_SIG_LEN 8
#define MBOX_MAX_MSG_LEN 128
#include "bset.h"
#include <linux/console.h>
+#include <linux/sched/clock.h>
#include <linux/random.h>
#include <linux/prefetch.h>
#include <linux/prefetch.h>
#include <linux/random.h>
#include <linux/rcupdate.h>
+#include <linux/sched/clock.h>
+#include <linux/rculist.h>
+
#include <trace/events/bcache.h>
/*
#include <linux/llist.h>
#include <linux/sched.h>
+#include <linux/sched/task_stack.h>
#include <linux/workqueue.h>
/*
#include <linux/blkdev.h>
#include <linux/sort.h>
+#include <linux/sched/clock.h>
static const char * const cache_replacement_policies[] = {
"lru",
#include <linux/module.h>
#include <linux/seq_file.h>
#include <linux/types.h>
+#include <linux/sched/clock.h>
#include "util.h"
#include <linux/blkdev.h>
#include <linux/errno.h>
-#include <linux/blkdev.h>
#include <linux/kernel.h>
+#include <linux/sched/clock.h>
#include <linux/llist.h>
#include <linux/ratelimit.h>
#include <linux/vmalloc.h>
#include <linux/delay.h>
#include <linux/kthread.h>
+#include <linux/sched/clock.h>
#include <trace/events/bcache.h>
/* Rate limiting */
#include <linux/device-mapper.h>
#include <linux/dm-io.h>
#include <linux/slab.h>
+#include <linux/sched/mm.h>
#include <linux/jiffies.h>
#include <linux/vmalloc.h>
#include <linux/shrinker.h>
down_read(&key->sem);
- ukp = user_key_payload(key);
+ ukp = user_key_payload_locked(key);
if (!ukp) {
up_read(&key->sem);
key_put(key);
#include <linux/module.h>
#include <linux/vmalloc.h>
#include <linux/miscdevice.h>
+#include <linux/sched/mm.h>
#include <linux/init.h>
#include <linux/wait.h>
#include <linux/slab.h>
#define CTR_FLAG_RAID10_USE_NEAR_SETS (1 << __CTR_FLAG_RAID10_USE_NEAR_SETS)
#define CTR_FLAG_JOURNAL_DEV (1 << __CTR_FLAG_JOURNAL_DEV)
+#define RESUME_STAY_FROZEN_FLAGS (CTR_FLAG_DELTA_DISKS | CTR_FLAG_DATA_OFFSET)
+
/*
* Definitions of various constructor flags to
* be used in checks of valid / invalid flags
else if (!strcasecmp(argv[0], "recover"))
set_bit(MD_RECOVERY_RECOVER, &mddev->recovery);
else {
- if (!strcasecmp(argv[0], "check"))
+ if (!strcasecmp(argv[0], "check")) {
set_bit(MD_RECOVERY_CHECK, &mddev->recovery);
- else if (!strcasecmp(argv[0], "repair")) {
+ set_bit(MD_RECOVERY_REQUESTED, &mddev->recovery);
+ set_bit(MD_RECOVERY_SYNC, &mddev->recovery);
+ } else if (!strcasecmp(argv[0], "repair")) {
set_bit(MD_RECOVERY_REQUESTED, &mddev->recovery);
set_bit(MD_RECOVERY_SYNC, &mddev->recovery);
} else
mddev->ro = 0;
mddev->in_sync = 0;
- clear_bit(MD_RECOVERY_FROZEN, &mddev->recovery);
+ /*
+ * Keep the RAID set frozen if reshape/rebuild flags are set.
+ * The RAID set is unfrozen once the next table load/resume,
+ * which clears the reshape/rebuild flags, occurs.
+ * This ensures that the constructor for the inactive table
+ * retrieves an up-to-date reshape_position.
+ */
+ if (!(rs->ctr_flags & RESUME_STAY_FROZEN_FLAGS))
+ clear_bit(MD_RECOVERY_FROZEN, &mddev->recovery);
if (mddev->suspended)
mddev_resume(mddev);
static struct target_type raid_target = {
.name = "raid",
- .version = {1, 10, 0},
+ .version = {1, 10, 1},
.module = THIS_MODULE,
.ctr = raid_ctr,
.dtr = raid_dtr,
#include <linux/init.h>
#include <linux/module.h>
#include <linux/mutex.h>
+#include <linux/sched/signal.h>
#include <linux/blkpg.h>
#include <linux/bio.h>
#include <linux/mempool.h>
*/
+#include <linux/sched/signal.h>
#include <linux/kthread.h>
#include <linux/blkdev.h>
#include <linux/badblocks.h>
#include <linux/rwsem.h>
#include <linux/device-mapper.h>
#include <linux/stacktrace.h>
+#include <linux/sched/task.h>
#define DM_MSG_PREFIX "block manager"
#include <linux/module.h>
#include <linux/seq_file.h>
#include <linux/ratelimit.h>
+#include <linux/sched/signal.h>
+
#include <trace/events/block.h>
+
#include "md.h"
#include "raid1.h"
#include "bitmap.h"
#include <linux/ratelimit.h>
#include <linux/nodemask.h>
#include <linux/flex_array.h>
+#include <linux/sched/signal.h>
+
#include <trace/events/block.h>
#include "md.h"
#include <linux/vmalloc.h>
#include <linux/delay.h>
#include <linux/spinlock.h>
-#include <linux/sched.h>
+#include <linux/sched/signal.h>
#include <linux/kthread.h>
#include "dvb_ca_en50221.h"
#define pr_fmt(fmt) "dvb_demux: " fmt
-#include <linux/sched.h>
+#include <linux/sched/signal.h>
#include <linux/spinlock.h>
#include <linux/slab.h>
#include <linux/vmalloc.h>
#include <linux/string.h>
#include <linux/kernel.h>
-#include <linux/sched.h>
+#include <linux/sched/signal.h>
#include <linux/wait.h>
#include <linux/slab.h>
#include <linux/poll.h>
*
* The actual DAP implementation may be restricted to only one of the modes.
* A compiler warning or error will be generated if the DAP implementation
-* overides or cannot handle the mode defined below.
-*
+* overrides or cannot handle the mode defined below.
*/
#ifndef DRXDAP_SINGLE_MASTER
#define DRXDAP_SINGLE_MASTER 1
*
* This maximum size may be restricted by the actual DAP implementation.
* A compiler warning or error will be generated if the DAP implementation
-* overides or cannot handle the chunksize defined below.
+* overrides or cannot handle the chunksize defined below.
*
* Beware that the DAP uses DRXDAP_MAX_WCHUNKSIZE to create a temporary data
* buffer. Do not undefine or choose too large, unless your system is able to
*
* This maximum size may be restricted by the actual DAP implementation.
* A compiler warning or error will be generated if the DAP implementation
-* overides or cannot handle the chunksize defined below.
-*
+* overrides or cannot handle the chunksize defined below.
*/
#ifndef DRXDAP_MAX_RCHUNKSIZE
#define DRXDAP_MAX_RCHUNKSIZE 60
#include <linux/moduleparam.h>
#include <linux/init.h>
#include <linux/delay.h>
-#include <linux/sched.h>
+#include <linux/sched/signal.h>
#include <linux/fs.h>
#include <linux/pci.h>
#include <linux/interrupt.h>
#include <media/tveeprom.h>
#include <media/i2c/saa7115.h>
#include "tuner-xc2028.h"
+#include <uapi/linux/sched/types.h>
/* If you have already X v4l cards, then set this to X. This way
the device numbers stay matched. Example: you have a WinTV card
* using information provided by Jiun-Kuei Jung @ AVerMedia.
*/
-#include <asm/byteorder.h>
+#include <linux/module.h>
+#include <linux/init.h>
#include <linux/delay.h>
-#include <linux/device.h>
+#include <linux/sched/signal.h>
#include <linux/fs.h>
+#include <linux/pci.h>
+#include <linux/interrupt.h>
+#include <linux/spinlock.h>
#include <linux/i2c.h>
#include <linux/i2c-algo-bit.h>
-#include <linux/init.h>
-#include <linux/interrupt.h>
-#include <linux/ivtv.h>
-#include <linux/kernel.h>
-#include <linux/kthread.h>
#include <linux/list.h>
-#include <linux/module.h>
-#include <linux/mutex.h>
+#include <linux/unistd.h>
#include <linux/pagemap.h>
-#include <linux/pci.h>
#include <linux/scatterlist.h>
-#include <linux/sched.h>
+#include <linux/kthread.h>
+#include <linux/mutex.h>
#include <linux/slab.h>
-#include <linux/spinlock.h>
#include <linux/uaccess.h>
-#include <linux/unistd.h>
+#include <asm/byteorder.h>
-#include <media/drv-intf/cx2341x.h>
-#include <media/i2c/ir-kbd-i2c.h>
-#include <media/tuner.h>
+#include <linux/dvb/video.h>
+#include <linux/dvb/audio.h>
#include <media/v4l2-common.h>
+#include <media/v4l2-ioctl.h>
#include <media/v4l2-ctrls.h>
#include <media/v4l2-device.h>
#include <media/v4l2-fh.h>
-#include <media/v4l2-ioctl.h>
+#include <media/tuner.h>
+#include <media/drv-intf/cx2341x.h>
+#include <media/i2c/ir-kbd-i2c.h>
+
+#include <linux/ivtv.h>
/* Memory layout */
#define IVTV_ENCODER_OFFSET 0x00000000
*/
#include <linux/kernel.h>
+#include <linux/sched/signal.h>
#include <linux/module.h>
#include <linux/slab.h>
#include <linux/vmalloc.h>
#include <linux/module.h>
#include <linux/pci.h>
#include <linux/string.h>
+#include <linux/sched/signal.h>
#include "dmxdev.h"
#include "dvbdev.h"
* thread context, ACK the interrupt, and move on. -- BenC */
#include <linux/kernel.h>
+#include <linux/sched/signal.h>
#include "solo6x10.h"
#include <linux/module.h>
#include <linux/vmalloc.h>
#include <linux/ktime.h>
+#include <linux/sched/signal.h>
#include <linux/interrupt.h>
#include <linux/proc_fs.h>
#include <linux/delay.h>
#include <linux/videodev2.h>
#include <linux/v4l2-dv-timings.h>
+#include <linux/sched/signal.h>
+
#include <media/v4l2-common.h>
#include <media/v4l2-event.h>
#include <media/v4l2-dv-timings.h>
#include <linux/errno.h>
#include <linux/kernel.h>
+#include <linux/sched/signal.h>
#include <linux/delay.h>
#include <linux/videodev2.h>
#include <linux/v4l2-dv-timings.h>
/**
* vsp1_du_setup_lif - Setup the output part of the VSP pipeline
* @dev: the VSP device
- * @width: output frame width in pixels
- * @height: output frame height in pixels
+ * @cfg: the LIF configuration
*
- * Configure the output part of VSP DRM pipeline for the given frame @width and
- * @height. This sets up formats on the BRU source pad, the WPF0 sink and source
- * pads, and the LIF sink pad.
+ * Configure the output part of VSP DRM pipeline for the given frame @cfg.width
+ * and @cfg.height. This sets up formats on the BRU source pad, the WPF0 sink
+ * and source pads, and the LIF sink pad.
*
* As the media bus code on the BRU source pad is conditioned by the
* configuration of the BRU sink 0 pad, we also set up the formats on all BRU
*
* Return 0 on success or a negative error code on failure.
*/
-int vsp1_du_setup_lif(struct device *dev, unsigned int width,
- unsigned int height)
+int vsp1_du_setup_lif(struct device *dev, const struct vsp1_du_lif_config *cfg)
{
struct vsp1_device *vsp1 = dev_get_drvdata(dev);
struct vsp1_pipeline *pipe = &vsp1->drm->pipe;
unsigned int i;
int ret;
- dev_dbg(vsp1->dev, "%s: configuring LIF with format %ux%u\n",
- __func__, width, height);
-
- if (width == 0 || height == 0) {
- /* Zero width or height means the CRTC is being disabled, stop
+ if (!cfg) {
+ /* NULL configuration means the CRTC is being disabled, stop
* the pipeline and turn the light off.
*/
ret = vsp1_pipeline_stop(pipe);
return 0;
}
+ dev_dbg(vsp1->dev, "%s: configuring LIF with format %ux%u\n",
+ __func__, cfg->width, cfg->height);
+
/* Configure the format at the BRU sinks and propagate it through the
* pipeline.
*/
for (i = 0; i < bru->entity.source_pad; ++i) {
format.pad = i;
- format.format.width = width;
- format.format.height = height;
+ format.format.width = cfg->width;
+ format.format.height = cfg->height;
format.format.code = MEDIA_BUS_FMT_ARGB8888_1X32;
format.format.field = V4L2_FIELD_NONE;
}
format.pad = bru->entity.source_pad;
- format.format.width = width;
- format.format.height = height;
+ format.format.width = cfg->width;
+ format.format.height = cfg->height;
format.format.code = MEDIA_BUS_FMT_ARGB8888_1X32;
format.format.field = V4L2_FIELD_NONE;
/* Verify that the format at the output of the pipeline matches the
* requested frame size and media bus code.
*/
- if (format.format.width != width || format.format.height != height ||
+ if (format.format.width != cfg->width ||
+ format.format.height != cfg->height ||
format.format.code != MEDIA_BUS_FMT_ARGB8888_1X32) {
dev_dbg(vsp1->dev, "%s: format mismatch\n", __func__);
return -EPIPE;
#include <linux/module.h>
#include <linux/kernel.h>
-#include <linux/sched.h>
+#include <linux/sched/signal.h>
#include <linux/errno.h>
#include <linux/ioctl.h>
#include <linux/fs.h>
return -ERESTARTSYS;
ir = irctls[iminor(inode)];
+ mutex_unlock(&lirc_dev_lock);
+
if (!ir) {
retval = -ENODEV;
goto error;
}
error:
- mutex_unlock(&lirc_dev_lock);
-
nonseekable_open(inode, file);
return retval;
{
u8 tolerance, config;
struct nvt_dev *nvt = dev->priv;
+ unsigned long flags;
int i;
/* hardcode the tolerance to 10% */
tolerance = DIV_ROUND_UP(count, 10);
- spin_lock(&nvt->lock);
+ spin_lock_irqsave(&nvt->lock, flags);
nvt_clear_cir_wake_fifo(nvt);
nvt_cir_wake_reg_write(nvt, count, CIR_WAKE_FIFO_CMP_DEEP);
nvt_cir_wake_reg_write(nvt, config, CIR_WAKE_IRCON);
- spin_unlock(&nvt->lock);
+ spin_unlock_irqrestore(&nvt->lock, flags);
}
static ssize_t wakeup_data_show(struct device *dev,
{
int rc;
struct rc_map *rc_map;
+ u64 rc_type;
if (!dev->map_name)
return -EINVAL;
if (rc)
return rc;
- if (dev->change_protocol) {
- u64 rc_type = (1ll << rc_map->rc_type);
+ rc_type = BIT_ULL(rc_map->rc_type);
+ if (dev->change_protocol) {
rc = dev->change_protocol(dev, &rc_type);
if (rc < 0)
goto out_table;
dev->enabled_protocols = rc_type;
}
+ if (dev->driver_type == RC_DRIVER_IR_RAW)
+ ir_raw_load_modules(&rc_type);
+
set_bit(EV_KEY, dev->input_dev->evbit);
set_bit(EV_REP, dev->input_dev->evbit);
set_bit(EV_MSC, dev->input_dev->evbit);
dev->input_name ?: "Unspecified device", path ?: "N/A");
kfree(path);
- if (dev->driver_type != RC_DRIVER_IR_RAW_TX) {
- rc = rc_setup_rx_device(dev);
- if (rc)
- goto out_dev;
- }
-
if (dev->driver_type == RC_DRIVER_IR_RAW ||
dev->driver_type == RC_DRIVER_IR_RAW_TX) {
if (!raw_init) {
}
rc = ir_raw_event_register(dev);
if (rc < 0)
- goto out_rx;
+ goto out_dev;
+ }
+
+ if (dev->driver_type != RC_DRIVER_IR_RAW_TX) {
+ rc = rc_setup_rx_device(dev);
+ if (rc)
+ goto out_raw;
}
/* Allow the RC sysfs nodes to be accessible */
return 0;
-out_rx:
- rc_free_rx_device(dev);
+out_raw:
+ ir_raw_event_unregister(dev);
out_dev:
device_del(&dev->dev);
out_unlock:
ir_raw_event_handle(serial_ir.rcdev);
}
+/* Needed by serial_ir_probe() */
+static int serial_ir_tx(struct rc_dev *dev, unsigned int *txbuf,
+ unsigned int count);
+static int serial_ir_tx_duty_cycle(struct rc_dev *dev, u32 cycle);
+static int serial_ir_tx_carrier(struct rc_dev *dev, u32 carrier);
+static int serial_ir_open(struct rc_dev *rcdev);
+static void serial_ir_close(struct rc_dev *rcdev);
+
static int serial_ir_probe(struct platform_device *dev)
{
+ struct rc_dev *rcdev;
int i, nlow, nhigh, result;
+ rcdev = devm_rc_allocate_device(&dev->dev, RC_DRIVER_IR_RAW);
+ if (!rcdev)
+ return -ENOMEM;
+
+ if (hardware[type].send_pulse && hardware[type].send_space)
+ rcdev->tx_ir = serial_ir_tx;
+ if (hardware[type].set_send_carrier)
+ rcdev->s_tx_carrier = serial_ir_tx_carrier;
+ if (hardware[type].set_duty_cycle)
+ rcdev->s_tx_duty_cycle = serial_ir_tx_duty_cycle;
+
+ switch (type) {
+ case IR_HOMEBREW:
+ rcdev->input_name = "Serial IR type home-brew";
+ break;
+ case IR_IRDEO:
+ rcdev->input_name = "Serial IR type IRdeo";
+ break;
+ case IR_IRDEO_REMOTE:
+ rcdev->input_name = "Serial IR type IRdeo remote";
+ break;
+ case IR_ANIMAX:
+ rcdev->input_name = "Serial IR type AnimaX";
+ break;
+ case IR_IGOR:
+ rcdev->input_name = "Serial IR type IgorPlug";
+ break;
+ }
+
+ rcdev->input_phys = KBUILD_MODNAME "/input0";
+ rcdev->input_id.bustype = BUS_HOST;
+ rcdev->input_id.vendor = 0x0001;
+ rcdev->input_id.product = 0x0001;
+ rcdev->input_id.version = 0x0100;
+ rcdev->open = serial_ir_open;
+ rcdev->close = serial_ir_close;
+ rcdev->dev.parent = &serial_ir.pdev->dev;
+ rcdev->allowed_protocols = RC_BIT_ALL_IR_DECODER;
+ rcdev->driver_name = KBUILD_MODNAME;
+ rcdev->map_name = RC_MAP_RC6_MCE;
+ rcdev->min_timeout = 1;
+ rcdev->timeout = IR_DEFAULT_TIMEOUT;
+ rcdev->max_timeout = 10 * IR_DEFAULT_TIMEOUT;
+ rcdev->rx_resolution = 250000;
+
+ serial_ir.rcdev = rcdev;
+
+ setup_timer(&serial_ir.timeout_timer, serial_ir_timeout,
+ (unsigned long)&serial_ir);
+
result = devm_request_irq(&dev->dev, irq, serial_ir_irq_handler,
share_irq ? IRQF_SHARED : 0,
KBUILD_MODNAME, &hardware);
return -EBUSY;
}
- setup_timer(&serial_ir.timeout_timer, serial_ir_timeout,
- (unsigned long)&serial_ir);
-
result = hardware_init_port();
if (result < 0)
return result;
sense ? "low" : "high");
dev_dbg(&dev->dev, "Interrupt %d, port %04x obtained\n", irq, io);
- return 0;
+
+ return devm_rc_register_device(&dev->dev, rcdev);
}
static int serial_ir_open(struct rc_dev *rcdev)
static int __init serial_ir_init_module(void)
{
- struct rc_dev *rcdev;
int result;
switch (type) {
sense = !!sense;
result = serial_ir_init();
- if (result)
- return result;
-
- rcdev = devm_rc_allocate_device(&serial_ir.pdev->dev, RC_DRIVER_IR_RAW);
- if (!rcdev) {
- result = -ENOMEM;
- goto serial_cleanup;
- }
-
- if (hardware[type].send_pulse && hardware[type].send_space)
- rcdev->tx_ir = serial_ir_tx;
- if (hardware[type].set_send_carrier)
- rcdev->s_tx_carrier = serial_ir_tx_carrier;
- if (hardware[type].set_duty_cycle)
- rcdev->s_tx_duty_cycle = serial_ir_tx_duty_cycle;
-
- switch (type) {
- case IR_HOMEBREW:
- rcdev->input_name = "Serial IR type home-brew";
- break;
- case IR_IRDEO:
- rcdev->input_name = "Serial IR type IRdeo";
- break;
- case IR_IRDEO_REMOTE:
- rcdev->input_name = "Serial IR type IRdeo remote";
- break;
- case IR_ANIMAX:
- rcdev->input_name = "Serial IR type AnimaX";
- break;
- case IR_IGOR:
- rcdev->input_name = "Serial IR type IgorPlug";
- break;
- }
-
- rcdev->input_phys = KBUILD_MODNAME "/input0";
- rcdev->input_id.bustype = BUS_HOST;
- rcdev->input_id.vendor = 0x0001;
- rcdev->input_id.product = 0x0001;
- rcdev->input_id.version = 0x0100;
- rcdev->open = serial_ir_open;
- rcdev->close = serial_ir_close;
- rcdev->dev.parent = &serial_ir.pdev->dev;
- rcdev->allowed_protocols = RC_BIT_ALL_IR_DECODER;
- rcdev->driver_name = KBUILD_MODNAME;
- rcdev->map_name = RC_MAP_RC6_MCE;
- rcdev->min_timeout = 1;
- rcdev->timeout = IR_DEFAULT_TIMEOUT;
- rcdev->max_timeout = 10 * IR_DEFAULT_TIMEOUT;
- rcdev->rx_resolution = 250000;
-
- serial_ir.rcdev = rcdev;
-
- result = rc_register_device(rcdev);
-
if (!result)
return 0;
-serial_cleanup:
+
serial_ir_exit();
return result;
}
static void __exit serial_ir_exit_module(void)
{
del_timer_sync(&serial_ir.timeout_timer);
- rc_unregister_device(serial_ir.rcdev);
serial_ir_exit();
}
#include <linux/mm.h>
#include <linux/vmalloc.h>
#include <linux/firmware.h>
+#include <linux/sched/signal.h>
#define FIRMWARE "cpia2/stv0672_vp4.bin"
MODULE_FIRMWARE(FIRMWARE);
struct dw2102_state {
u8 initialized;
u8 last_lock;
+ u8 data[MAX_XFER_SIZE + 4];
struct i2c_client *i2c_client_demod;
struct i2c_client *i2c_client_tuner;
int num)
{
struct dvb_usb_device *d = i2c_get_adapdata(adap);
- u8 obuf[0x40], ibuf[0x40];
+ struct dw2102_state *state;
if (!d)
return -ENODEV;
+
+ state = d->priv;
+
if (mutex_lock_interruptible(&d->i2c_mutex) < 0)
return -EAGAIN;
+ if (mutex_lock_interruptible(&d->data_mutex) < 0) {
+ mutex_unlock(&d->i2c_mutex);
+ return -EAGAIN;
+ }
switch (num) {
case 1:
switch (msg[0].addr) {
case SU3000_STREAM_CTRL:
- obuf[0] = msg[0].buf[0] + 0x36;
- obuf[1] = 3;
- obuf[2] = 0;
- if (dvb_usb_generic_rw(d, obuf, 3, ibuf, 0, 0) < 0)
+ state->data[0] = msg[0].buf[0] + 0x36;
+ state->data[1] = 3;
+ state->data[2] = 0;
+ if (dvb_usb_generic_rw(d, state->data, 3,
+ state->data, 0, 0) < 0)
err("i2c transfer failed.");
break;
case DW2102_RC_QUERY:
- obuf[0] = 0x10;
- if (dvb_usb_generic_rw(d, obuf, 1, ibuf, 2, 0) < 0)
+ state->data[0] = 0x10;
+ if (dvb_usb_generic_rw(d, state->data, 1,
+ state->data, 2, 0) < 0)
err("i2c transfer failed.");
- msg[0].buf[1] = ibuf[0];
- msg[0].buf[0] = ibuf[1];
+ msg[0].buf[1] = state->data[0];
+ msg[0].buf[0] = state->data[1];
break;
default:
/* always i2c write*/
- obuf[0] = 0x08;
- obuf[1] = msg[0].addr;
- obuf[2] = msg[0].len;
+ state->data[0] = 0x08;
+ state->data[1] = msg[0].addr;
+ state->data[2] = msg[0].len;
- memcpy(&obuf[3], msg[0].buf, msg[0].len);
+ memcpy(&state->data[3], msg[0].buf, msg[0].len);
- if (dvb_usb_generic_rw(d, obuf, msg[0].len + 3,
- ibuf, 1, 0) < 0)
+ if (dvb_usb_generic_rw(d, state->data, msg[0].len + 3,
+ state->data, 1, 0) < 0)
err("i2c transfer failed.");
}
break;
case 2:
/* always i2c read */
- obuf[0] = 0x09;
- obuf[1] = msg[0].len;
- obuf[2] = msg[1].len;
- obuf[3] = msg[0].addr;
- memcpy(&obuf[4], msg[0].buf, msg[0].len);
-
- if (dvb_usb_generic_rw(d, obuf, msg[0].len + 4,
- ibuf, msg[1].len + 1, 0) < 0)
+ state->data[0] = 0x09;
+ state->data[1] = msg[0].len;
+ state->data[2] = msg[1].len;
+ state->data[3] = msg[0].addr;
+ memcpy(&state->data[4], msg[0].buf, msg[0].len);
+
+ if (dvb_usb_generic_rw(d, state->data, msg[0].len + 4,
+ state->data, msg[1].len + 1, 0) < 0)
err("i2c transfer failed.");
- memcpy(msg[1].buf, &ibuf[1], msg[1].len);
+ memcpy(msg[1].buf, &state->data[1], msg[1].len);
break;
default:
warn("more than 2 i2c messages at a time is not handled yet.");
break;
}
+ mutex_unlock(&d->data_mutex);
mutex_unlock(&d->i2c_mutex);
return num;
}
static int su3000_power_ctrl(struct dvb_usb_device *d, int i)
{
struct dw2102_state *state = (struct dw2102_state *)d->priv;
- u8 obuf[] = {0xde, 0};
+ int ret = 0;
info("%s: %d, initialized %d", __func__, i, state->initialized);
if (i && !state->initialized) {
+ mutex_lock(&d->data_mutex);
+
+ state->data[0] = 0xde;
+ state->data[1] = 0;
+
state->initialized = 1;
/* reset board */
- return dvb_usb_generic_rw(d, obuf, 2, NULL, 0, 0);
+ ret = dvb_usb_generic_rw(d, state->data, 2, NULL, 0, 0);
+ mutex_unlock(&d->data_mutex);
}
- return 0;
+ return ret;
}
static int su3000_read_mac_address(struct dvb_usb_device *d, u8 mac[6])
return 0;
}
-static int su3000_frontend_attach(struct dvb_usb_adapter *d)
+static int su3000_frontend_attach(struct dvb_usb_adapter *adap)
{
- u8 obuf[3] = { 0xe, 0x80, 0 };
- u8 ibuf[] = { 0 };
+ struct dvb_usb_device *d = adap->dev;
+ struct dw2102_state *state = d->priv;
+
+ mutex_lock(&d->data_mutex);
+
+ state->data[0] = 0xe;
+ state->data[1] = 0x80;
+ state->data[2] = 0;
- if (dvb_usb_generic_rw(d->dev, obuf, 3, ibuf, 1, 0) < 0)
+ if (dvb_usb_generic_rw(d, state->data, 3, state->data, 1, 0) < 0)
err("command 0x0e transfer failed.");
- obuf[0] = 0xe;
- obuf[1] = 0x02;
- obuf[2] = 1;
+ state->data[0] = 0xe;
+ state->data[1] = 0x02;
+ state->data[2] = 1;
- if (dvb_usb_generic_rw(d->dev, obuf, 3, ibuf, 1, 0) < 0)
+ if (dvb_usb_generic_rw(d, state->data, 3, state->data, 1, 0) < 0)
err("command 0x0e transfer failed.");
msleep(300);
- obuf[0] = 0xe;
- obuf[1] = 0x83;
- obuf[2] = 0;
+ state->data[0] = 0xe;
+ state->data[1] = 0x83;
+ state->data[2] = 0;
- if (dvb_usb_generic_rw(d->dev, obuf, 3, ibuf, 1, 0) < 0)
+ if (dvb_usb_generic_rw(d, state->data, 3, state->data, 1, 0) < 0)
err("command 0x0e transfer failed.");
- obuf[0] = 0xe;
- obuf[1] = 0x83;
- obuf[2] = 1;
+ state->data[0] = 0xe;
+ state->data[1] = 0x83;
+ state->data[2] = 1;
- if (dvb_usb_generic_rw(d->dev, obuf, 3, ibuf, 1, 0) < 0)
+ if (dvb_usb_generic_rw(d, state->data, 3, state->data, 1, 0) < 0)
err("command 0x0e transfer failed.");
- obuf[0] = 0x51;
+ state->data[0] = 0x51;
- if (dvb_usb_generic_rw(d->dev, obuf, 1, ibuf, 1, 0) < 0)
+ if (dvb_usb_generic_rw(d, state->data, 1, state->data, 1, 0) < 0)
err("command 0x51 transfer failed.");
- d->fe_adap[0].fe = dvb_attach(ds3000_attach, &su3000_ds3000_config,
- &d->dev->i2c_adap);
- if (d->fe_adap[0].fe == NULL)
+ mutex_unlock(&d->data_mutex);
+
+ adap->fe_adap[0].fe = dvb_attach(ds3000_attach, &su3000_ds3000_config,
+ &d->i2c_adap);
+ if (adap->fe_adap[0].fe == NULL)
return -EIO;
- if (dvb_attach(ts2020_attach, d->fe_adap[0].fe,
+ if (dvb_attach(ts2020_attach, adap->fe_adap[0].fe,
&dw2104_ts2020_config,
- &d->dev->i2c_adap)) {
+ &d->i2c_adap)) {
info("Attached DS3000/TS2020!");
return 0;
}
return -EIO;
}
-static int t220_frontend_attach(struct dvb_usb_adapter *d)
+static int t220_frontend_attach(struct dvb_usb_adapter *adap)
{
- u8 obuf[3] = { 0xe, 0x87, 0 };
- u8 ibuf[] = { 0 };
+ struct dvb_usb_device *d = adap->dev;
+ struct dw2102_state *state = d->priv;
+
+ mutex_lock(&d->data_mutex);
- if (dvb_usb_generic_rw(d->dev, obuf, 3, ibuf, 1, 0) < 0)
+ state->data[0] = 0xe;
+ state->data[1] = 0x87;
+ state->data[2] = 0x0;
+
+ if (dvb_usb_generic_rw(d, state->data, 3, state->data, 1, 0) < 0)
err("command 0x0e transfer failed.");
- obuf[0] = 0xe;
- obuf[1] = 0x86;
- obuf[2] = 1;
+ state->data[0] = 0xe;
+ state->data[1] = 0x86;
+ state->data[2] = 1;
- if (dvb_usb_generic_rw(d->dev, obuf, 3, ibuf, 1, 0) < 0)
+ if (dvb_usb_generic_rw(d, state->data, 3, state->data, 1, 0) < 0)
err("command 0x0e transfer failed.");
- obuf[0] = 0xe;
- obuf[1] = 0x80;
- obuf[2] = 0;
+ state->data[0] = 0xe;
+ state->data[1] = 0x80;
+ state->data[2] = 0;
- if (dvb_usb_generic_rw(d->dev, obuf, 3, ibuf, 1, 0) < 0)
+ if (dvb_usb_generic_rw(d, state->data, 3, state->data, 1, 0) < 0)
err("command 0x0e transfer failed.");
msleep(50);
- obuf[0] = 0xe;
- obuf[1] = 0x80;
- obuf[2] = 1;
+ state->data[0] = 0xe;
+ state->data[1] = 0x80;
+ state->data[2] = 1;
- if (dvb_usb_generic_rw(d->dev, obuf, 3, ibuf, 1, 0) < 0)
+ if (dvb_usb_generic_rw(d, state->data, 3, state->data, 1, 0) < 0)
err("command 0x0e transfer failed.");
- obuf[0] = 0x51;
+ state->data[0] = 0x51;
- if (dvb_usb_generic_rw(d->dev, obuf, 1, ibuf, 1, 0) < 0)
+ if (dvb_usb_generic_rw(d, state->data, 1, state->data, 1, 0) < 0)
err("command 0x51 transfer failed.");
- d->fe_adap[0].fe = dvb_attach(cxd2820r_attach, &cxd2820r_config,
- &d->dev->i2c_adap, NULL);
- if (d->fe_adap[0].fe != NULL) {
- if (dvb_attach(tda18271_attach, d->fe_adap[0].fe, 0x60,
- &d->dev->i2c_adap, &tda18271_config)) {
+ mutex_unlock(&d->data_mutex);
+
+ adap->fe_adap[0].fe = dvb_attach(cxd2820r_attach, &cxd2820r_config,
+ &d->i2c_adap, NULL);
+ if (adap->fe_adap[0].fe != NULL) {
+ if (dvb_attach(tda18271_attach, adap->fe_adap[0].fe, 0x60,
+ &d->i2c_adap, &tda18271_config)) {
info("Attached TDA18271HD/CXD2820R!");
return 0;
}
return -EIO;
}
-static int m88rs2000_frontend_attach(struct dvb_usb_adapter *d)
+static int m88rs2000_frontend_attach(struct dvb_usb_adapter *adap)
{
- u8 obuf[] = { 0x51 };
- u8 ibuf[] = { 0 };
+ struct dvb_usb_device *d = adap->dev;
+ struct dw2102_state *state = d->priv;
+
+ mutex_lock(&d->data_mutex);
- if (dvb_usb_generic_rw(d->dev, obuf, 1, ibuf, 1, 0) < 0)
+ state->data[0] = 0x51;
+
+ if (dvb_usb_generic_rw(d, state->data, 1, state->data, 1, 0) < 0)
err("command 0x51 transfer failed.");
- d->fe_adap[0].fe = dvb_attach(m88rs2000_attach, &s421_m88rs2000_config,
- &d->dev->i2c_adap);
+ mutex_unlock(&d->data_mutex);
- if (d->fe_adap[0].fe == NULL)
+ adap->fe_adap[0].fe = dvb_attach(m88rs2000_attach,
+ &s421_m88rs2000_config,
+ &d->i2c_adap);
+
+ if (adap->fe_adap[0].fe == NULL)
return -EIO;
- if (dvb_attach(ts2020_attach, d->fe_adap[0].fe,
+ if (dvb_attach(ts2020_attach, adap->fe_adap[0].fe,
&dw2104_ts2020_config,
- &d->dev->i2c_adap)) {
+ &d->i2c_adap)) {
info("Attached RS2000/TS2020!");
return 0;
}
{
struct dvb_usb_device *d = adap->dev;
struct dw2102_state *state = d->priv;
- u8 obuf[3] = { 0xe, 0x80, 0 };
- u8 ibuf[] = { 0 };
struct i2c_adapter *i2c_adapter;
struct i2c_client *client;
struct i2c_board_info board_info;
struct m88ds3103_platform_data m88ds3103_pdata = {};
struct ts2020_config ts2020_config = {};
- if (dvb_usb_generic_rw(d, obuf, 3, ibuf, 1, 0) < 0)
+ mutex_lock(&d->data_mutex);
+
+ state->data[0] = 0xe;
+ state->data[1] = 0x80;
+ state->data[2] = 0x0;
+
+ if (dvb_usb_generic_rw(d, state->data, 3, state->data, 1, 0) < 0)
err("command 0x0e transfer failed.");
- obuf[0] = 0xe;
- obuf[1] = 0x02;
- obuf[2] = 1;
+ state->data[0] = 0xe;
+ state->data[1] = 0x02;
+ state->data[2] = 1;
- if (dvb_usb_generic_rw(d, obuf, 3, ibuf, 1, 0) < 0)
+ if (dvb_usb_generic_rw(d, state->data, 3, state->data, 1, 0) < 0)
err("command 0x0e transfer failed.");
msleep(300);
- obuf[0] = 0xe;
- obuf[1] = 0x83;
- obuf[2] = 0;
+ state->data[0] = 0xe;
+ state->data[1] = 0x83;
+ state->data[2] = 0;
- if (dvb_usb_generic_rw(d, obuf, 3, ibuf, 1, 0) < 0)
+ if (dvb_usb_generic_rw(d, state->data, 3, state->data, 1, 0) < 0)
err("command 0x0e transfer failed.");
- obuf[0] = 0xe;
- obuf[1] = 0x83;
- obuf[2] = 1;
+ state->data[0] = 0xe;
+ state->data[1] = 0x83;
+ state->data[2] = 1;
- if (dvb_usb_generic_rw(d, obuf, 3, ibuf, 1, 0) < 0)
+ if (dvb_usb_generic_rw(d, state->data, 3, state->data, 1, 0) < 0)
err("command 0x0e transfer failed.");
- obuf[0] = 0x51;
+ state->data[0] = 0x51;
- if (dvb_usb_generic_rw(d, obuf, 1, ibuf, 1, 0) < 0)
+ if (dvb_usb_generic_rw(d, state->data, 1, state->data, 1, 0) < 0)
err("command 0x51 transfer failed.");
+ mutex_unlock(&d->data_mutex);
+
/* attach demod */
m88ds3103_pdata.clk = 27000000;
m88ds3103_pdata.i2c_wr_max = 33;
#define MODULE_NAME "cpia1"
#include <linux/input.h>
+#include <linux/sched/signal.h>
+
#include "gspca.h"
MODULE_AUTHOR("Hans de Goede <hdegoede@redhat.com>");
#include <linux/init.h>
#include <linux/module.h>
#include <linux/moduleparam.h>
-#include <linux/sched.h>
+#include <linux/sched/mm.h>
#include <linux/slab.h>
#include <linux/interrupt.h>
*/
#include <linux/workqueue.h>
-#include <linux/sched.h>
+#include <linux/sched/signal.h>
+#include <linux/sched/mm.h>
#include <linux/pid.h>
#include <linux/mm.h>
#include <linux/moduleparam.h>
#include <linux/export.h>
#include <linux/kernel.h>
#include <linux/bitmap.h>
-#include <linux/sched.h>
+#include <linux/sched/signal.h>
#include <linux/poll.h>
#include <linux/pid.h>
#include <linux/fs.h>
#include <linux/slab.h>
#include <linux/idr.h>
#include <linux/pci.h>
+#include <linux/sched/task.h>
+
#include <asm/cputable.h>
#include <misc/cxl-base.h>
#include <linux/spinlock.h>
#include <linux/sched.h>
+#include <linux/sched/clock.h>
#include <linux/slab.h>
#include <linux/mutex.h>
#include <linux/mm.h>
#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/device.h>
+#include <linux/capability.h>
#include <linux/jiffies.h>
#include <linux/i2c.h>
#include <linux/mutex.h>
#include <linux/pci.h>
#include <linux/string.h>
#include <linux/fs.h>
-#include <linux/sched.h>
+#include <linux/sched/signal.h>
#include <linux/wait.h>
#include <linux/delay.h>
#include <linux/atomic.h>
*
*/
-#include <linux/sched.h>
+#include <linux/sched/signal.h>
#include "ibmasm.h"
#include "dot_command.h"
#include <linux/delay.h>
#include <linux/kthread.h>
#include <linux/module.h>
+#include <linux/sched/task.h>
+
#include <asm/sections.h>
#define v1printk(a...) do { \
#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
#include <linux/kernel.h>
+#include <linux/sched/signal.h>
#include <linux/dmi.h>
#include <linux/module.h>
#include <linux/types.h>
*/
#include "lkdtm.h"
#include <linux/slab.h>
+#include <linux/sched.h>
/*
* This tries to stay within the next largest power-of-2 kmalloc cache
#include "lkdtm.h"
#include <linux/slab.h>
#include <linux/vmalloc.h>
+#include <linux/sched/task_stack.h>
#include <linux/mman.h>
#include <linux/uaccess.h>
#include <asm/cacheflush.h>
#include <linux/module.h>
#include <linux/device.h>
#include <linux/kernel.h>
-#include <linux/sched.h>
+#include <linux/sched/signal.h>
#include <linux/init.h>
#include <linux/errno.h>
#include <linux/slab.h>
*
*/
-#include <linux/sched.h>
+#include <linux/sched/signal.h>
#include <linux/wait.h>
#include <linux/delay.h>
#include <linux/slab.h>
#include <linux/init.h>
#include <linux/ioctl.h>
#include <linux/cdev.h>
-#include <linux/sched.h>
+#include <linux/sched/signal.h>
#include <linux/uuid.h>
#include <linux/compat.h>
#include <linux/jiffies.h>
*
*/
#include <linux/kthread.h>
+#include <linux/sched/signal.h>
+
#include "cosm_main.h"
/*
#include <linux/delay.h>
#include <linux/reboot.h>
#include <linux/kthread.h>
+#include <linux/sched/signal.h>
+
#include "../cosm/cosm_main.h"
#define COSM_SCIF_MAX_RETRIES 10
#ifndef SCIF_MAIN_H
#define SCIF_MAIN_H
-#include <linux/sched.h>
+#include <linux/sched/signal.h>
#include <linux/pci.h>
#include <linux/miscdevice.h>
#include <linux/dmaengine.h>
*/
#include <linux/dma_remapping.h>
#include <linux/pagemap.h>
+#include <linux/sched/mm.h>
+#include <linux/sched/signal.h>
+
#include "scif_main.h"
#include "scif_map.h"
static int vop_find_vqs(struct virtio_device *dev, unsigned nvqs,
struct virtqueue *vqs[],
vq_callback_t *callbacks[],
- const char * const names[])
+ const char * const names[], struct irq_affinity *desc)
{
struct _vop_vdev *vdev = to_vopvdev(dev);
struct vop_device *vpdev = vdev->vpdev;
int write, unsigned long *paddr, int *pageshift)
{
pgd_t *pgdp;
- pmd_t *pmdp;
+ p4d_t *p4dp;
pud_t *pudp;
+ pmd_t *pmdp;
pte_t pte;
pgdp = pgd_offset(vma->vm_mm, vaddr);
if (unlikely(pgd_none(*pgdp)))
goto err;
- pudp = pud_offset(pgdp, vaddr);
+ p4dp = p4d_offset(pgdp, vaddr);
+ if (unlikely(p4d_none(*p4dp)))
+ goto err;
+
+ pudp = pud_offset(p4dp, vaddr);
if (unlikely(pud_none(*pudp)))
goto err;
#include <linux/io.h>
#include <linux/of.h>
#include <linux/platform_device.h>
-#include <linux/sched.h>
+#include <linux/sched/signal.h>
#include <linux/slab.h>
#include <linux/syscore_ops.h>
#include <linux/vexpress.h>
#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/sched.h>
+#include <linux/cred.h>
#include <linux/slab.h>
#include "vmci_queue_pair.h"
#include <linux/module.h>
#include <linux/sched.h>
#include <linux/slab.h>
+#include <linux/rculist.h>
#include "vmci_driver.h"
#include "vmci_event.h"
#include <linux/module.h>
#include <linux/mutex.h>
#include <linux/sched.h>
+#include <linux/cred.h>
#include <linux/slab.h>
#include <linux/file.h>
#include <linux/init.h>
#include <linux/hash.h>
#include <linux/types.h>
#include <linux/rculist.h>
+#include <linux/completion.h>
#include "vmci_resource.h"
#include "vmci_driver.h"
#include <linux/kernel.h>
#include <linux/sched.h>
+#include <uapi/linux/sched/types.h>
#include <linux/kthread.h>
#include <linux/export.h>
#include <linux/wait.h>
*/
#include <linux/clk.h>
#include <linux/slab.h>
+#include <linux/sched/task_stack.h>
#include <linux/interrupt.h>
#include <linux/module.h>
#include <linux/mtd/partitions.h>
#include <linux/sched.h>
#include <linux/slab.h>
#include <linux/mm.h>
+#include <linux/nmi.h>
#include <linux/types.h>
#include <linux/mtd/mtd.h>
#include <linux/mtd/nand.h>
}
/*
- * Send write disble instruction to the chip.
+ * Send write disable instruction to the chip.
*/
static inline int write_disable(struct spi_nor *nor)
{
#include <linux/mtd/mtd.h>
-#include <linux/sched.h>
+#include <linux/sched/signal.h>
static inline int mtdtest_relax(void)
{
if (err)
return ERR_PTR(err);
- err = vfs_getattr(&path, &stat);
+ err = vfs_getattr(&path, &stat, STATX_TYPE, AT_STATX_SYNC_AS_STAT);
path_put(&path);
if (err)
return ERR_PTR(err);
if (error)
return ERR_PTR(error);
- error = vfs_getattr(&path, &stat);
+ error = vfs_getattr(&path, &stat, STATX_TYPE, AT_STATX_SYNC_AS_STAT);
path_put(&path);
if (error)
return ERR_PTR(error);
/* Initialize the device entry points */
ether_setup(bond_dev);
+ bond_dev->max_mtu = ETH_MAX_MTU;
bond_dev->netdev_ops = &bond_netdev_ops;
bond_dev->ethtool_ops = &bond_ethtool_ops;
#include <linux/rcupdate.h>
#include <linux/ctype.h>
#include <linux/inet.h>
+#include <linux/sched/signal.h>
+
#include <net/bonding.h>
static int bond_option_active_slave_set(struct bonding *bond,
#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/device.h>
-#include <linux/sched.h>
+#include <linux/sched/signal.h>
#include <linux/fs.h>
#include <linux/types.h>
#include <linux/string.h>
goto err;
/* Get the TX virtio ring. This is a "guest side vring". */
- err = vdev->config->find_vqs(vdev, 1, &cfv->vq_tx, &vq_cbs, &names);
+ err = vdev->config->find_vqs(vdev, 1, &cfv->vq_tx, &vq_cbs, &names,
+ NULL);
if (err)
goto err;
#define FLEXCAN_QUIRK_BROKEN_ERR_STATE BIT(1) /* [TR]WRN_INT not connected */
#define FLEXCAN_QUIRK_DISABLE_RXFG BIT(2) /* Disable RX FIFO Global mask */
#define FLEXCAN_QUIRK_ENABLE_EACEN_RRS BIT(3) /* Enable EACEN and RRS bit in ctrl2 */
-#define FLEXCAN_QUIRK_DISABLE_MECR BIT(4) /* Disble Memory error detection */
+#define FLEXCAN_QUIRK_DISABLE_MECR BIT(4) /* Disable Memory error detection */
#define FLEXCAN_QUIRK_USE_OFF_TIMESTAMP BIT(5) /* Use timestamp based offloading */
/* Structure of the message buffer */
*/
#include <linux/firmware.h>
-#include <linux/sched.h>
+#include <linux/sched/signal.h>
#include <asm/div64.h>
#include <asm/io.h>
rc = usb_control_msg(interface_to_usbdev(intf),
usb_sndctrlpipe(interface_to_usbdev(intf), 0),
GS_USB_BREQ_MODE,
- USB_DIR_OUT|USB_TYPE_VENDOR|USB_RECIP_INTERFACE,
+ USB_DIR_OUT | USB_TYPE_VENDOR | USB_RECIP_INTERFACE,
gsdev->channel,
0,
dm,
rc = usb_control_msg(interface_to_usbdev(intf),
usb_sndctrlpipe(interface_to_usbdev(intf), 0),
GS_USB_BREQ_BITTIMING,
- USB_DIR_OUT|USB_TYPE_VENDOR|USB_RECIP_INTERFACE,
+ USB_DIR_OUT | USB_TYPE_VENDOR | USB_RECIP_INTERFACE,
dev->channel,
0,
dbt,
hf,
urb->transfer_dma);
-
if (rc == -ENODEV) {
netif_device_detach(netdev);
} else {
rc = usb_control_msg(interface_to_usbdev(intf),
usb_rcvctrlpipe(interface_to_usbdev(intf), 0),
GS_USB_BREQ_BT_CONST,
- USB_DIR_IN|USB_TYPE_VENDOR|USB_RECIP_INTERFACE,
+ USB_DIR_IN | USB_TYPE_VENDOR | USB_RECIP_INTERFACE,
channel,
0,
bt_const,
struct gs_usb *dev;
int rc = -ENOMEM;
unsigned int icount, i;
- struct gs_host_config hconf = {
- .byte_order = 0x0000beef,
- };
- struct gs_device_config dconf;
+ struct gs_host_config *hconf;
+ struct gs_device_config *dconf;
+
+ hconf = kmalloc(sizeof(*hconf), GFP_KERNEL);
+ if (!hconf)
+ return -ENOMEM;
+
+ hconf->byte_order = 0x0000beef;
/* send host config */
rc = usb_control_msg(interface_to_usbdev(intf),
usb_sndctrlpipe(interface_to_usbdev(intf), 0),
GS_USB_BREQ_HOST_FORMAT,
- USB_DIR_OUT|USB_TYPE_VENDOR|USB_RECIP_INTERFACE,
+ USB_DIR_OUT | USB_TYPE_VENDOR | USB_RECIP_INTERFACE,
1,
intf->altsetting[0].desc.bInterfaceNumber,
- &hconf,
- sizeof(hconf),
+ hconf,
+ sizeof(*hconf),
1000);
+ kfree(hconf);
+
if (rc < 0) {
dev_err(&intf->dev, "Couldn't send data format (err=%d)\n",
rc);
return rc;
}
+ dconf = kmalloc(sizeof(*dconf), GFP_KERNEL);
+ if (!dconf)
+ return -ENOMEM;
+
/* read device config */
rc = usb_control_msg(interface_to_usbdev(intf),
usb_rcvctrlpipe(interface_to_usbdev(intf), 0),
GS_USB_BREQ_DEVICE_CONFIG,
- USB_DIR_IN|USB_TYPE_VENDOR|USB_RECIP_INTERFACE,
+ USB_DIR_IN | USB_TYPE_VENDOR | USB_RECIP_INTERFACE,
1,
intf->altsetting[0].desc.bInterfaceNumber,
- &dconf,
- sizeof(dconf),
+ dconf,
+ sizeof(*dconf),
1000);
if (rc < 0) {
dev_err(&intf->dev, "Couldn't get device config: (err=%d)\n",
rc);
+ kfree(dconf);
return rc;
}
- icount = dconf.icount + 1;
+ icount = dconf->icount + 1;
dev_info(&intf->dev, "Configuring for %d interfaces\n", icount);
if (icount > GS_MAX_INTF) {
dev_err(&intf->dev,
"Driver cannot handle more that %d CAN interfaces\n",
GS_MAX_INTF);
+ kfree(dconf);
return -EINVAL;
}
dev = kzalloc(sizeof(*dev), GFP_KERNEL);
- if (!dev)
+ if (!dev) {
+ kfree(dconf);
return -ENOMEM;
+ }
+
init_usb_anchor(&dev->rx_submitted);
atomic_set(&dev->active_channels, 0);
dev->udev = interface_to_usbdev(intf);
for (i = 0; i < icount; i++) {
- dev->canch[i] = gs_make_candev(i, intf, &dconf);
+ dev->canch[i] = gs_make_candev(i, intf, dconf);
if (IS_ERR_OR_NULL(dev->canch[i])) {
/* save error code to return later */
rc = PTR_ERR(dev->canch[i]);
gs_destroy_candev(dev->canch[i]);
usb_kill_anchored_urbs(&dev->rx_submitted);
+ kfree(dconf);
kfree(dev);
return rc;
}
dev->canch[i]->parent = dev;
}
+ kfree(dconf);
+
return 0;
}
for (i = 0; i < MAX_TX_URBS; i++)
priv->tx_contexts[i].echo_index = MAX_TX_URBS;
- priv->cmd_msg_buffer = kzalloc(sizeof(struct usb_8dev_cmd_msg),
- GFP_KERNEL);
+ priv->cmd_msg_buffer = devm_kzalloc(&intf->dev, sizeof(struct usb_8dev_cmd_msg),
+ GFP_KERNEL);
if (!priv->cmd_msg_buffer)
goto cleanup_candev;
if (err) {
netdev_err(netdev,
"couldn't register CAN device: %d\n", err);
- goto cleanup_cmd_msg_buffer;
+ goto cleanup_candev;
}
err = usb_8dev_cmd_version(priv, &version);
cleanup_unregister_candev:
unregister_netdev(priv->netdev);
-cleanup_cmd_msg_buffer:
- kfree(priv->cmd_msg_buffer);
-
cleanup_candev:
free_candev(netdev);
return ret;
}
-static void __exit dec_lance_remove(struct device *bdev)
-{
- struct net_device *dev = dev_get_drvdata(bdev);
- resource_size_t start, len;
-
- unregister_netdev(dev);
- start = to_tc_dev(bdev)->resource.start;
- len = to_tc_dev(bdev)->resource.end - start + 1;
- release_mem_region(start, len);
- free_netdev(dev);
-}
-
/* Find all the lance cards on the system and initialize them */
static int __init dec_lance_platform_probe(void)
{
#ifdef CONFIG_TC
static int dec_lance_tc_probe(struct device *dev);
-static int __exit dec_lance_tc_remove(struct device *dev);
+static int dec_lance_tc_remove(struct device *dev);
static const struct tc_device_id dec_lance_tc_table[] = {
{ "DEC ", "PMAD-AA " },
.name = "declance",
.bus = &tc_bus_type,
.probe = dec_lance_tc_probe,
- .remove = __exit_p(dec_lance_tc_remove),
+ .remove = dec_lance_tc_remove,
},
};
return status;
}
-static int __exit dec_lance_tc_remove(struct device *dev)
+static void dec_lance_remove(struct device *bdev)
+{
+ struct net_device *dev = dev_get_drvdata(bdev);
+ resource_size_t start, len;
+
+ unregister_netdev(dev);
+ start = to_tc_dev(bdev)->resource.start;
+ len = to_tc_dev(bdev)->resource.end - start + 1;
+ release_mem_region(start, len);
+ free_netdev(dev);
+}
+
+static int dec_lance_tc_remove(struct device *dev)
{
put_device(dev);
dec_lance_remove(dev);
static int xgbe_set_ext_mii_mode(struct xgbe_prv_data *pdata, unsigned int port,
enum xgbe_mdio_mode mode)
{
- unsigned int reg_val = 0;
+ unsigned int reg_val = XGMAC_IOREAD(pdata, MAC_MDIOCL22R);
switch (mode) {
case XGBE_MDIO_MODE_CL22:
hw_if->disable_tx(pdata);
hw_if->disable_rx(pdata);
+ phy_if->phy_stop(pdata);
+
xgbe_free_irqs(pdata);
xgbe_napi_disable(pdata, 1);
- phy_if->phy_stop(pdata);
-
hw_if->exit(pdata);
channel = pdata->channel;
pdata->phy.duplex = DUPLEX_UNKNOWN;
pdata->phy.autoneg = AUTONEG_ENABLE;
pdata->phy.advertising = pdata->phy.supported;
+
+ return;
}
pdata->phy.advertising &= ~ADVERTISED_Autoneg;
!phy_data->sfp_phy_avail)
return 0;
+ /* Set the proper MDIO mode for the PHY */
+ ret = pdata->hw_if.set_ext_mii_mode(pdata, phy_data->mdio_addr,
+ phy_data->phydev_mode);
+ if (ret) {
+ netdev_err(pdata->netdev,
+ "mdio port/clause not compatible (%u/%u)\n",
+ phy_data->mdio_addr, phy_data->phydev_mode);
+ return ret;
+ }
+
/* Create and connect to the PHY device */
phydev = get_phy_device(phy_data->mii, phy_data->mdio_addr,
(phy_data->phydev_mode == XGBE_MDIO_MODE_CL45));
if (ret)
return ret;
+ /* Set the proper MDIO mode for the re-driver */
+ if (phy_data->redrv && !phy_data->redrv_if) {
+ ret = pdata->hw_if.set_ext_mii_mode(pdata, phy_data->redrv_addr,
+ XGBE_MDIO_MODE_CL22);
+ if (ret) {
+ netdev_err(pdata->netdev,
+ "redriver mdio port not compatible (%u)\n",
+ phy_data->redrv_addr);
+ return ret;
+ }
+ }
+
/* Start in highest supported mode */
xgbe_phy_set_mode(pdata, phy_data->start_mode);
pdata->clk = devm_clk_get(&pdev->dev, NULL);
if (IS_ERR(pdata->clk)) {
+ /* Abort if the clock is defined but couldn't be retrived.
+ * Always abort if the clock is missing on DT system as
+ * the driver can't cope with this case.
+ */
+ if (PTR_ERR(pdata->clk) != -ENOENT || dev->of_node)
+ return PTR_ERR(pdata->clk);
/* Firmware may have set up the clock already. */
dev_info(dev, "clocks have been setup already\n");
}
static bool platform_bgmac_clk_enabled(struct bgmac *bgmac)
{
- if ((bgmac_idm_read(bgmac, BCMA_IOCTL) &
- (BCMA_IOCTL_CLK | BCMA_IOCTL_FGC)) != BCMA_IOCTL_CLK)
+ if ((bgmac_idm_read(bgmac, BCMA_IOCTL) & BGMAC_CLK_EN) != BGMAC_CLK_EN)
return false;
if (bgmac_idm_read(bgmac, BCMA_RESET_CTL) & BCMA_RESET_CTL_RESET)
return false;
static void platform_bgmac_clk_enable(struct bgmac *bgmac, u32 flags)
{
- bgmac_idm_write(bgmac, BCMA_IOCTL,
- (BCMA_IOCTL_CLK | BCMA_IOCTL_FGC | flags));
- bgmac_idm_read(bgmac, BCMA_IOCTL);
+ u32 val;
- bgmac_idm_write(bgmac, BCMA_RESET_CTL, 0);
- bgmac_idm_read(bgmac, BCMA_RESET_CTL);
- udelay(1);
+ /* The Reset Control register only contains a single bit to show if the
+ * controller is currently in reset. Do a sanity check here, just in
+ * case the bootloader happened to leave the device in reset.
+ */
+ val = bgmac_idm_read(bgmac, BCMA_RESET_CTL);
+ if (val) {
+ bgmac_idm_write(bgmac, BCMA_RESET_CTL, 0);
+ bgmac_idm_read(bgmac, BCMA_RESET_CTL);
+ udelay(1);
+ }
- bgmac_idm_write(bgmac, BCMA_IOCTL, (BCMA_IOCTL_CLK | flags));
+ val = bgmac_idm_read(bgmac, BCMA_IOCTL);
+ /* Some bits of BCMA_IOCTL set by HW/ATF and should not change */
+ val |= flags & ~(BGMAC_AWCACHE | BGMAC_ARCACHE | BGMAC_AWUSER |
+ BGMAC_ARUSER);
+ val |= BGMAC_CLK_EN;
+ bgmac_idm_write(bgmac, BCMA_IOCTL, val);
bgmac_idm_read(bgmac, BCMA_IOCTL);
udelay(1);
}
static int bgmac_set_mac_address(struct net_device *net_dev, void *addr)
{
struct bgmac *bgmac = netdev_priv(net_dev);
+ struct sockaddr *sa = addr;
int ret;
ret = eth_prepare_mac_addr_change(net_dev, addr);
if (ret < 0)
return ret;
- bgmac_write_mac_address(bgmac, (u8 *)addr);
+
+ ether_addr_copy(net_dev->dev_addr, sa->sa_data);
+ bgmac_write_mac_address(bgmac, net_dev->dev_addr);
+
eth_commit_mac_addr_change(net_dev, addr);
return 0;
}
/* BCMA GMAC core specific IO Control (BCMA_IOCTL) flags */
#define BGMAC_BCMA_IOCTL_SW_CLKEN 0x00000004 /* PHY Clock Enable */
#define BGMAC_BCMA_IOCTL_SW_RESET 0x00000008 /* PHY Reset */
+/* The IOCTL values appear to be different in NS, NSP, and NS2, and do not match
+ * the values directly above
+ */
+#define BGMAC_CLK_EN BIT(0)
+#define BGMAC_RESERVED_0 BIT(1)
+#define BGMAC_SOURCE_SYNC_MODE_EN BIT(2)
+#define BGMAC_DEST_SYNC_MODE_EN BIT(3)
+#define BGMAC_TX_CLK_OUT_INVERT_EN BIT(4)
+#define BGMAC_DIRECT_GMII_MODE BIT(5)
+#define BGMAC_CLK_250_SEL BIT(6)
+#define BGMAC_AWCACHE (0xf << 7)
+#define BGMAC_RESERVED_1 (0x1f << 11)
+#define BGMAC_ARCACHE (0xf << 16)
+#define BGMAC_AWUSER (0x3f << 20)
+#define BGMAC_ARUSER (0x3f << 26)
+#define BGMAC_RESERVED BIT(31)
/* BCMA GMAC core specific IO status (BCMA_IOST) flags */
#define BGMAC_BCMA_IOST_ATTACHED 0x00000800
return err;
}
-static int __exit sbmac_remove(struct platform_device *pldev)
+static int sbmac_remove(struct platform_device *pldev)
{
struct net_device *dev = platform_get_drvdata(pldev);
struct sbmac_softc *sc = netdev_priv(dev);
static struct platform_driver sbmac_driver = {
.probe = sbmac_probe,
- .remove = __exit_p(sbmac_remove),
+ .remove = sbmac_remove,
.driver = {
.name = sbmac_string,
},
#include <linux/moduleparam.h>
#include <linux/stringify.h>
#include <linux/kernel.h>
+#include <linux/sched/signal.h>
#include <linux/types.h>
#include <linux/compiler.h>
#include <linux/slab.h>
#ifndef _OCTEON_MAIN_H_
#define _OCTEON_MAIN_H_
+#include <linux/sched/signal.h>
+
#if BITS_PER_LONG == 32
#define CVM_CAST64(v) ((long long)(v))
#elif BITS_PER_LONG == 64
#define TX_FORCE_S 13
#define TX_FORCE_V(x) ((x) << TX_FORCE_S)
+#define T6_TX_FORCE_S 20
+#define T6_TX_FORCE_V(x) ((x) << T6_TX_FORCE_S)
+#define T6_TX_FORCE_F T6_TX_FORCE_V(1U)
+
enum {
ULP_TX_MEM_READ = 2,
ULP_TX_MEM_WRITE = 3,
#define T4FW_VERSION_MAJOR 0x01
#define T4FW_VERSION_MINOR 0x10
-#define T4FW_VERSION_MICRO 0x1A
+#define T4FW_VERSION_MICRO 0x21
#define T4FW_VERSION_BUILD 0x00
#define T4FW_MIN_VERSION_MAJOR 0x01
#define T5FW_VERSION_MAJOR 0x01
#define T5FW_VERSION_MINOR 0x10
-#define T5FW_VERSION_MICRO 0x1A
+#define T5FW_VERSION_MICRO 0x21
#define T5FW_VERSION_BUILD 0x00
#define T5FW_MIN_VERSION_MAJOR 0x00
#define T6FW_VERSION_MAJOR 0x01
#define T6FW_VERSION_MINOR 0x10
-#define T6FW_VERSION_MICRO 0x1A
+#define T6FW_VERSION_MICRO 0x21
#define T6FW_VERSION_BUILD 0x00
#define T6FW_MIN_VERSION_MAJOR 0x00
#define PPOD_PAGES_MAX 4
struct cxgbi_pagepod {
struct cxgbi_pagepod_hdr hdr;
- u64 addr[PPOD_PAGES_MAX + 1];
+ __be64 addr[PPOD_PAGES_MAX + 1];
};
/* ddp tag format
return err;
}
-static int __exit ftgmac100_remove(struct platform_device *pdev)
+static int ftgmac100_remove(struct platform_device *pdev)
{
struct net_device *netdev;
struct ftgmac100 *priv;
static struct platform_driver ftgmac100_driver = {
.probe = ftgmac100_probe,
- .remove = __exit_p(ftgmac100_remove),
+ .remove = ftgmac100_remove,
.driver = {
.name = DRV_NAME,
.of_match_table = ftgmac100_of_match,
return err;
}
-static int __exit ftmac100_remove(struct platform_device *pdev)
+static int ftmac100_remove(struct platform_device *pdev)
{
struct net_device *netdev;
struct ftmac100 *priv;
static struct platform_driver ftmac100_driver = {
.probe = ftmac100_probe,
- .remove = __exit_p(ftmac100_remove),
+ .remove = ftmac100_remove,
.driver = {
.name = DRV_NAME,
},
#define IXGBE_MAX_FRAME_BUILD_SKB \
(SKB_WITH_OVERHEAD(IXGBE_RXBUFFER_2K) - IXGBE_SKB_PAD)
#else
-#define IGB_MAX_FRAME_BUILD_SKB IXGBE_RXBUFFER_2K
+#define IXGBE_MAX_FRAME_BUILD_SKB IXGBE_RXBUFFER_2K
#endif
/*
struct ixgbe_adapter *adapter,
struct ixgbe_ring *tx_ring);
u32 ixgbe_rss_indir_tbl_entries(struct ixgbe_adapter *adapter);
+void ixgbe_store_key(struct ixgbe_adapter *adapter);
void ixgbe_store_reta(struct ixgbe_adapter *adapter);
s32 ixgbe_negotiate_fc(struct ixgbe_hw *hw, u32 adv_reg, u32 lp_reg,
u32 adv_sym, u32 adv_asm, u32 lp_sym, u32 lp_asm);
}
/* Fill out the rss hash key */
- if (key)
+ if (key) {
memcpy(adapter->rss_key, key, ixgbe_get_rxfh_key_size(netdev));
+ ixgbe_store_key(adapter);
+ }
ixgbe_store_reta(adapter);
return 512;
}
+/**
+ * ixgbe_store_key - Write the RSS key to HW
+ * @adapter: device handle
+ *
+ * Write the RSS key stored in adapter.rss_key to HW.
+ */
+void ixgbe_store_key(struct ixgbe_adapter *adapter)
+{
+ struct ixgbe_hw *hw = &adapter->hw;
+ int i;
+
+ for (i = 0; i < 10; i++)
+ IXGBE_WRITE_REG(hw, IXGBE_RSSRK(i), adapter->rss_key[i]);
+}
+
/**
* ixgbe_store_reta - Write the RETA table to HW
* @adapter: device handle
static void ixgbe_setup_reta(struct ixgbe_adapter *adapter)
{
- struct ixgbe_hw *hw = &adapter->hw;
u32 i, j;
u32 reta_entries = ixgbe_rss_indir_tbl_entries(adapter);
u16 rss_i = adapter->ring_feature[RING_F_RSS].indices;
rss_i = 4;
/* Fill out hash function seeds */
- for (i = 0; i < 10; i++)
- IXGBE_WRITE_REG(hw, IXGBE_RSSRK(i), adapter->rss_key[i]);
+ ixgbe_store_key(adapter);
/* Fill out redirection table */
memset(adapter->rss_indir_tbl, 0, sizeof(adapter->rss_indir_tbl));
if (adapter->flags2 & IXGBE_FLAG2_RSC_ENABLED)
set_bit(__IXGBE_RX_3K_BUFFER, &rx_ring->state);
- if (max_frame > (ETH_FRAME_LEN + ETH_FCS_LEN))
+ if ((max_frame > (ETH_FRAME_LEN + ETH_FCS_LEN)) ||
+ (max_frame > IXGBE_MAX_FRAME_BUILD_SKB))
set_bit(__IXGBE_RX_3K_BUFFER, &rx_ring->state);
#endif
}
mlxsw_sp_vr_lpm_tree_check(struct mlxsw_sp *mlxsw_sp, struct mlxsw_sp_vr *vr,
struct mlxsw_sp_prefix_usage *req_prefix_usage)
{
- struct mlxsw_sp_lpm_tree *lpm_tree;
+ struct mlxsw_sp_lpm_tree *lpm_tree = vr->lpm_tree;
+ struct mlxsw_sp_lpm_tree *new_tree;
+ int err;
- if (mlxsw_sp_prefix_usage_eq(req_prefix_usage,
- &vr->lpm_tree->prefix_usage))
+ if (mlxsw_sp_prefix_usage_eq(req_prefix_usage, &lpm_tree->prefix_usage))
return 0;
- lpm_tree = mlxsw_sp_lpm_tree_get(mlxsw_sp, req_prefix_usage,
+ new_tree = mlxsw_sp_lpm_tree_get(mlxsw_sp, req_prefix_usage,
vr->proto, false);
- if (IS_ERR(lpm_tree)) {
+ if (IS_ERR(new_tree)) {
/* We failed to get a tree according to the required
* prefix usage. However, the current tree might be still good
* for us if our requirement is subset of the prefixes used
* in the tree.
*/
if (mlxsw_sp_prefix_usage_subset(req_prefix_usage,
- &vr->lpm_tree->prefix_usage))
+ &lpm_tree->prefix_usage))
return 0;
- return PTR_ERR(lpm_tree);
+ return PTR_ERR(new_tree);
}
- mlxsw_sp_vr_lpm_tree_unbind(mlxsw_sp, vr);
- mlxsw_sp_lpm_tree_put(mlxsw_sp, vr->lpm_tree);
+ /* Prevent packet loss by overwriting existing binding */
+ vr->lpm_tree = new_tree;
+ err = mlxsw_sp_vr_lpm_tree_bind(mlxsw_sp, vr);
+ if (err)
+ goto err_tree_bind;
+ mlxsw_sp_lpm_tree_put(mlxsw_sp, lpm_tree);
+
+ return 0;
+
+err_tree_bind:
vr->lpm_tree = lpm_tree;
- return mlxsw_sp_vr_lpm_tree_bind(mlxsw_sp, vr);
+ mlxsw_sp_lpm_tree_put(mlxsw_sp, new_tree);
+ return err;
}
static struct mlxsw_sp_vr *mlxsw_sp_vr_get(struct mlxsw_sp *mlxsw_sp,
txbuf->real_len = pkt_len;
dma_sync_single_for_device(&nn->pdev->dev, rxbuf->dma_addr + pkt_off,
- pkt_len, DMA_TO_DEVICE);
+ pkt_len, DMA_BIDIRECTIONAL);
/* Build TX descriptor */
txd = &tx_ring->txds[wr_idx];
dma_sync_single_for_cpu(&nn->pdev->dev,
rxbuf->dma_addr + pkt_off,
- pkt_len, DMA_FROM_DEVICE);
+ pkt_len, DMA_BIDIRECTIONAL);
act = nfp_net_run_xdp(xdp_prog, rxbuf->frag + data_off,
pkt_len);
switch (act) {
nfp_net_write_mac_addr(nn);
nn_writel(nn, NFP_NET_CFG_MTU, nn->netdev->mtu);
- nn_writel(nn, NFP_NET_CFG_FLBUFSZ, nn->fl_bufsz);
+ nn_writel(nn, NFP_NET_CFG_FLBUFSZ,
+ nn->fl_bufsz - NFP_NET_RX_BUF_NON_DATA);
/* Enable device */
new_ctrl |= NFP_NET_CFG_CTRL_ENABLE;
struct ib_mac_iocb_rsp {
u8 opcode; /* 0x20 */
u8 flags1;
-#define IB_MAC_IOCB_RSP_OI 0x01 /* Overide intr delay */
-#define IB_MAC_IOCB_RSP_I 0x02 /* Disble Intr Generation */
+#define IB_MAC_IOCB_RSP_OI 0x01 /* Override intr delay */
+#define IB_MAC_IOCB_RSP_I 0x02 /* Disable Intr Generation */
#define IB_MAC_CSUM_ERR_MASK 0x1c /* A mask to use for csum errs */
#define IB_MAC_IOCB_RSP_TE 0x04 /* Checksum error */
#define IB_MAC_IOCB_RSP_NU 0x08 /* No checksum rcvd */
return err;
}
-static int __exit sgiseeq_remove(struct platform_device *pdev)
+static int sgiseeq_remove(struct platform_device *pdev)
{
struct net_device *dev = platform_get_drvdata(pdev);
struct sgiseeq_private *sp = netdev_priv(dev);
static struct platform_driver sgiseeq_driver = {
.probe = sgiseeq_probe,
- .remove = __exit_p(sgiseeq_remove),
+ .remove = sgiseeq_remove,
.driver = {
.name = "sgiseeq",
}
static int efx_ef10_link_piobufs(struct efx_nic *efx)
{
struct efx_ef10_nic_data *nic_data = efx->nic_data;
- _MCDI_DECLARE_BUF(inbuf,
- max(MC_CMD_LINK_PIOBUF_IN_LEN,
- MC_CMD_UNLINK_PIOBUF_IN_LEN));
+ MCDI_DECLARE_BUF(inbuf, MC_CMD_LINK_PIOBUF_IN_LEN);
struct efx_channel *channel;
struct efx_tx_queue *tx_queue;
unsigned int offset, index;
BUILD_BUG_ON(MC_CMD_LINK_PIOBUF_OUT_LEN != 0);
BUILD_BUG_ON(MC_CMD_UNLINK_PIOBUF_OUT_LEN != 0);
- memset(inbuf, 0, sizeof(inbuf));
-
/* Link a buffer to each VI in the write-combining mapping */
for (index = 0; index < nic_data->n_piobufs; ++index) {
MCDI_SET_DWORD(inbuf, LINK_PIOBUF_IN_PIOBUF_HANDLE,
return 0;
fail:
+ /* inbuf was defined for MC_CMD_LINK_PIOBUF. We can use the same
+ * buffer for MC_CMD_UNLINK_PIOBUF because it's shorter.
+ */
+ BUILD_BUG_ON(MC_CMD_LINK_PIOBUF_IN_LEN < MC_CMD_UNLINK_PIOBUF_IN_LEN);
while (index--) {
MCDI_SET_DWORD(inbuf, UNLINK_PIOBUF_IN_TXQ_INSTANCE,
nic_data->pio_write_vi_base + index);
/* Modify IPv4 header if needed. */
ip->tot_len = 0;
ip->check = 0;
- ipv4_id = ip->id;
+ ipv4_id = ntohs(ip->id);
} else {
/* Modify IPv6 header if needed. */
struct ipv6hdr *ipv6 = ipv6_hdr(skb);
#include <linux/i2c.h>
#include <linux/mii.h>
#include <linux/slab.h>
+#include <linux/sched/signal.h>
+
#include "net_driver.h"
#include "bitfield.h"
#include "efx.h"
return 0;
}
-static int __exit meth_remove(struct platform_device *pdev)
+static int meth_remove(struct platform_device *pdev)
{
struct net_device *dev = platform_get_drvdata(pdev);
static struct platform_driver meth_driver = {
.probe = meth_probe,
- .remove = __exit_p(meth_remove),
+ .remove = meth_remove,
.driver = {
.name = "meth",
}
info = &geneve->info;
}
+ rcu_read_lock();
#if IS_ENABLED(CONFIG_IPV6)
if (info->mode & IP_TUNNEL_INFO_IPV6)
err = geneve6_xmit_skb(skb, dev, geneve, info);
else
#endif
err = geneve_xmit_skb(skb, dev, geneve, info);
+ rcu_read_unlock();
if (likely(!err))
return NETDEV_TX_OK;
if (ret)
goto out;
+ memset(&device_info, 0, sizeof(device_info));
+ device_info.ring_size = ring_size;
+ device_info.num_chn = nvdev->num_chn;
+ device_info.max_num_vrss_chns = nvdev->num_chn;
+
ndevctx->start_remove = true;
rndis_filter_device_remove(hdev, nvdev);
+ /* 'nvdev' has been freed in rndis_filter_device_remove() ->
+ * netvsc_device_remove () -> free_netvsc_device().
+ * We mustn't access it before it's re-created in
+ * rndis_filter_device_add() -> netvsc_device_add().
+ */
+
ndev->mtu = mtu;
- memset(&device_info, 0, sizeof(device_info));
- device_info.ring_size = ring_size;
- device_info.num_chn = nvdev->num_chn;
- device_info.max_num_vrss_chns = nvdev->num_chn;
rndis_filter_device_add(hdev, &device_info);
out:
#include <linux/dma/pxa-dma.h>
#include <linux/gpio.h>
#include <linux/slab.h>
+#include <linux/sched/clock.h>
#include <net/irda/irda.h>
#include <net/irda/irmod.h>
#include <linux/moduleparam.h>
#include <linux/kernel.h>
+#include <linux/sched/signal.h>
#include <linux/ktime.h>
#include <linux/types.h>
#include <linux/time.h>
#include <linux/module.h>
#include <linux/skbuff.h>
#include <linux/cache.h>
-#include <linux/sched.h>
+#include <linux/sched/signal.h>
#include <linux/types.h>
#include <linux/slab.h>
#include <linux/wait.h>
#include <linux/module.h>
#include <linux/kernel.h>
+#include <linux/sched/signal.h>
#include <linux/kmod.h>
#include <linux/init.h>
#include <linux/list.h>
#include <linux/uaccess.h>
#include <linux/bitops.h>
-#include <linux/sched.h>
+#include <linux/sched/signal.h>
#include <linux/string.h>
#include <linux/mm.h>
#include <linux/interrupt.h>
#include <linux/module.h>
#include <linux/skbuff.h>
#include <linux/cache.h>
-#include <linux/sched.h>
+#include <linux/sched/signal.h>
#include <linux/types.h>
#include <linux/slab.h>
#include <linux/wait.h>
#include <linux/module.h>
#include <linux/errno.h>
#include <linux/kernel.h>
+#include <linux/sched/signal.h>
#include <linux/major.h>
#include <linux/slab.h>
#include <linux/poll.h>
if (ret < 0)
goto out;
- asix_write_medium_mode(dev, AX88772_MEDIUM_DEFAULT, 0);
+ ret = asix_write_medium_mode(dev, AX88772_MEDIUM_DEFAULT, 0);
if (ret < 0)
goto out;
#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
-#include <linux/sched.h>
+#include <linux/sched/signal.h>
#include <linux/slab.h>
#include <linux/init.h>
#include <linux/delay.h>
*/
#include <linux/module.h>
+#include <linux/sched/signal.h>
#include <linux/netdevice.h>
#include <linux/ethtool.h>
#include <linux/etherdevice.h>
* at once, the weight is chosen so that the EWMA will be insensitive to short-
* term, transient changes in packet size.
*/
-DECLARE_EWMA(pkt_len, 1, 64)
+DECLARE_EWMA(pkt_len, 0, 64)
/* With mergeable buffers we align buffer address and use the low bits to
* encode its true size. Buffer size is up to 1 page so we need to align to
}
ret = vi->vdev->config->find_vqs(vi->vdev, total_vqs, vqs, callbacks,
- names);
+ names, NULL);
if (ret)
goto err_find;
src_port = udp_flow_src_port(dev_net(dev), skb, vxlan->cfg.port_min,
vxlan->cfg.port_max, true);
+ rcu_read_lock();
if (dst->sa.sa_family == AF_INET) {
struct vxlan_sock *sock4 = rcu_dereference(vxlan->vn4_sock);
struct rtable *rt;
dst_port, vni, &rt->dst,
rt->rt_flags);
if (err)
- return;
+ goto out_unlock;
} else if (info->key.tun_flags & TUNNEL_DONT_FRAGMENT) {
df = htons(IP_DF);
}
dst_port, vni, ndst,
rt6i_flags);
if (err)
- return;
+ goto out_unlock;
}
tos = ip_tunnel_ecn_encap(tos, old_iph, skb);
label, src_port, dst_port, !udp_sum);
#endif
}
+out_unlock:
+ rcu_read_unlock();
return;
drop:
return;
tx_error:
+ rcu_read_unlock();
if (err == -ELOOP)
dev->stats.collisions++;
else if (err == -ENETUNREACH)
#include <linux/module.h>
#include <linux/kernel.h>
-#include <linux/sched.h>
+#include <linux/sched/signal.h>
#include <linux/slab.h>
#include <linux/poll.h>
#include <linux/fs.h>
#include <linux/module.h>
#include <linux/firmware.h>
#include <linux/of.h>
+#include <linux/dmi.h>
+#include <linux/ctype.h>
#include <asm/byteorder.h>
#include "core.h"
return 0;
}
+static void ath10k_core_check_bdfext(const struct dmi_header *hdr, void *data)
+{
+ struct ath10k *ar = data;
+ const char *bdf_ext;
+ const char *magic = ATH10K_SMBIOS_BDF_EXT_MAGIC;
+ u8 bdf_enabled;
+ int i;
+
+ if (hdr->type != ATH10K_SMBIOS_BDF_EXT_TYPE)
+ return;
+
+ if (hdr->length != ATH10K_SMBIOS_BDF_EXT_LENGTH) {
+ ath10k_dbg(ar, ATH10K_DBG_BOOT,
+ "wrong smbios bdf ext type length (%d).\n",
+ hdr->length);
+ return;
+ }
+
+ bdf_enabled = *((u8 *)hdr + ATH10K_SMBIOS_BDF_EXT_OFFSET);
+ if (!bdf_enabled) {
+ ath10k_dbg(ar, ATH10K_DBG_BOOT, "bdf variant name not found.\n");
+ return;
+ }
+
+ /* Only one string exists (per spec) */
+ bdf_ext = (char *)hdr + hdr->length;
+
+ if (memcmp(bdf_ext, magic, strlen(magic)) != 0) {
+ ath10k_dbg(ar, ATH10K_DBG_BOOT,
+ "bdf variant magic does not match.\n");
+ return;
+ }
+
+ for (i = 0; i < strlen(bdf_ext); i++) {
+ if (!isascii(bdf_ext[i]) || !isprint(bdf_ext[i])) {
+ ath10k_dbg(ar, ATH10K_DBG_BOOT,
+ "bdf variant name contains non ascii chars.\n");
+ return;
+ }
+ }
+
+ /* Copy extension name without magic suffix */
+ if (strscpy(ar->id.bdf_ext, bdf_ext + strlen(magic),
+ sizeof(ar->id.bdf_ext)) < 0) {
+ ath10k_dbg(ar, ATH10K_DBG_BOOT,
+ "bdf variant string is longer than the buffer can accommodate (variant: %s)\n",
+ bdf_ext);
+ return;
+ }
+
+ ath10k_dbg(ar, ATH10K_DBG_BOOT,
+ "found and validated bdf variant smbios_type 0x%x bdf %s\n",
+ ATH10K_SMBIOS_BDF_EXT_TYPE, bdf_ext);
+}
+
+static int ath10k_core_check_smbios(struct ath10k *ar)
+{
+ ar->id.bdf_ext[0] = '\0';
+ dmi_walk(ath10k_core_check_bdfext, ar);
+
+ if (ar->id.bdf_ext[0] == '\0')
+ return -ENODATA;
+
+ return 0;
+}
+
static int ath10k_download_and_run_otp(struct ath10k *ar)
{
u32 result, address = ar->hw_params.patch_load_addr;
case ATH10K_BD_IE_BOARD:
ret = ath10k_core_parse_bd_ie_board(ar, data, ie_len,
boardname);
+ if (ret == -ENOENT && ar->id.bdf_ext[0] != '\0') {
+ /* try default bdf if variant was not found */
+ char *s, *v = ",variant=";
+ char boardname2[100];
+
+ strlcpy(boardname2, boardname,
+ sizeof(boardname2));
+
+ s = strstr(boardname2, v);
+ if (s)
+ *s = '\0'; /* strip ",variant=%s" */
+
+ ret = ath10k_core_parse_bd_ie_board(ar, data,
+ ie_len,
+ boardname2);
+ }
+
if (ret == -ENOENT)
/* no match found, continue */
break;
static int ath10k_core_create_board_name(struct ath10k *ar, char *name,
size_t name_len)
{
+ /* strlen(',variant=') + strlen(ar->id.bdf_ext) */
+ char variant[9 + ATH10K_SMBIOS_BDF_EXT_STR_LENGTH] = { 0 };
+
if (ar->id.bmi_ids_valid) {
scnprintf(name, name_len,
"bus=%s,bmi-chip-id=%d,bmi-board-id=%d",
goto out;
}
+ if (ar->id.bdf_ext[0] != '\0')
+ scnprintf(variant, sizeof(variant), ",variant=%s",
+ ar->id.bdf_ext);
+
scnprintf(name, name_len,
- "bus=%s,vendor=%04x,device=%04x,subsystem-vendor=%04x,subsystem-device=%04x",
+ "bus=%s,vendor=%04x,device=%04x,subsystem-vendor=%04x,subsystem-device=%04x%s",
ath10k_bus_str(ar->hif.bus),
ar->id.vendor, ar->id.device,
- ar->id.subsystem_vendor, ar->id.subsystem_device);
-
+ ar->id.subsystem_vendor, ar->id.subsystem_device, variant);
out:
ath10k_dbg(ar, ATH10K_DBG_BOOT, "boot using board name '%s'\n", name);
goto err_free_firmware_files;
}
+ ret = ath10k_core_check_smbios(ar);
+ if (ret)
+ ath10k_dbg(ar, ATH10K_DBG_BOOT, "bdf variant name not set.\n");
+
ret = ath10k_core_fetch_board_file(ar);
if (ret) {
ath10k_err(ar, "failed to fetch board file: %d\n", ret);
#define ATH10K_NAPI_BUDGET 64
#define ATH10K_NAPI_QUOTA_LIMIT 60
+/* SMBIOS type containing Board Data File Name Extension */
+#define ATH10K_SMBIOS_BDF_EXT_TYPE 0xF8
+
+/* SMBIOS type structure length (excluding strings-set) */
+#define ATH10K_SMBIOS_BDF_EXT_LENGTH 0x9
+
+/* Offset pointing to Board Data File Name Extension */
+#define ATH10K_SMBIOS_BDF_EXT_OFFSET 0x8
+
+/* Board Data File Name Extension string length.
+ * String format: BDF_<Customer ID>_<Extension>\0
+ */
+#define ATH10K_SMBIOS_BDF_EXT_STR_LENGTH 0x20
+
+/* The magic used by QCA spec */
+#define ATH10K_SMBIOS_BDF_EXT_MAGIC "BDF_"
+
struct ath10k;
enum ath10k_bus {
bool bmi_ids_valid;
u8 bmi_board_id;
u8 bmi_chip_id;
+
+ char bdf_ext[ATH10K_SMBIOS_BDF_EXT_STR_LENGTH];
} id;
int fw_api;
#define ATH5K_TXQ_LEN_MAX (ATH_TXBUF / 4) /* bufs per queue */
#define ATH5K_TXQ_LEN_LOW (ATH5K_TXQ_LEN_MAX / 2) /* low mark */
-DECLARE_EWMA(beacon_rssi, 1024, 8)
+DECLARE_EWMA(beacon_rssi, 10, 8)
/* Driver state associated with an instance of a device */
struct ath5k_hw {
#include <linux/moduleparam.h>
#include <linux/inetdevice.h>
#include <linux/export.h>
+#include <linux/sched/signal.h>
#include "core.h"
#include "cfg80211.h"
#include <linux/etherdevice.h>
#include <linux/firmware.h>
#include <linux/workqueue.h>
-#include <linux/sched.h>
+#include <linux/sched/signal.h>
#include <linux/skbuff.h>
#include <linux/dma-mapping.h>
#include <linux/slab.h>
#include <linux/pci_ids.h>
#include <linux/netdevice.h>
#include <linux/interrupt.h>
-#include <linux/sched.h>
+#include <linux/sched/signal.h>
#include <linux/mmc/sdio.h>
#include <linux/mmc/sdio_ids.h>
#include <linux/mmc/sdio_func.h>
#include <linux/delay.h>
#include <linux/random.h>
#include <linux/wait.h>
-#include <linux/sched.h>
+#include <linux/sched/signal.h>
#include <linux/rtnetlink.h>
#include <linux/wireless.h>
#include <net/iw_handler.h>
#include <linux/slab.h>
#include <linux/types.h>
-#include <linux/sched.h>
+#include <linux/sched/signal.h>
#include <linux/ethtool.h>
#include <linux/if_arp.h>
#include <linux/module.h>
static int hwsim_new_radio_nl(struct sk_buff *msg, struct genl_info *info)
{
struct hwsim_new_radio_params param = { 0 };
+ const char *hwname = NULL;
param.reg_strict = info->attrs[HWSIM_ATTR_REG_STRICT_REG];
param.p2p_device = info->attrs[HWSIM_ATTR_SUPPORT_P2P_DEVICE];
if (info->attrs[HWSIM_ATTR_NO_VIF])
param.no_vif = true;
- if (info->attrs[HWSIM_ATTR_RADIO_NAME])
- param.hwname = nla_data(info->attrs[HWSIM_ATTR_RADIO_NAME]);
+ if (info->attrs[HWSIM_ATTR_RADIO_NAME]) {
+ hwname = kasprintf(GFP_KERNEL, "%.*s",
+ nla_len(info->attrs[HWSIM_ATTR_RADIO_NAME]),
+ (char *)nla_data(info->attrs[HWSIM_ATTR_RADIO_NAME]));
+ if (!hwname)
+ return -ENOMEM;
+ param.hwname = hwname;
+ }
if (info->attrs[HWSIM_ATTR_USE_CHANCTX])
param.use_chanctx = true;
s64 idx = -1;
const char *hwname = NULL;
- if (info->attrs[HWSIM_ATTR_RADIO_ID])
+ if (info->attrs[HWSIM_ATTR_RADIO_ID]) {
idx = nla_get_u32(info->attrs[HWSIM_ATTR_RADIO_ID]);
- else if (info->attrs[HWSIM_ATTR_RADIO_NAME])
- hwname = (void *)nla_data(info->attrs[HWSIM_ATTR_RADIO_NAME]);
- else
+ } else if (info->attrs[HWSIM_ATTR_RADIO_NAME]) {
+ hwname = kasprintf(GFP_KERNEL, "%.*s",
+ nla_len(info->attrs[HWSIM_ATTR_RADIO_NAME]),
+ (char *)nla_data(info->attrs[HWSIM_ATTR_RADIO_NAME]));
+ if (!hwname)
+ return -ENOMEM;
+ } else
return -EINVAL;
spin_lock_bh(&hwsim_radio_lock);
if (data->idx != idx)
continue;
} else {
- if (strcmp(hwname, wiphy_name(data->hw->wiphy)))
+ if (!hwname ||
+ strcmp(hwname, wiphy_name(data->hw->wiphy)))
continue;
}
spin_unlock_bh(&hwsim_radio_lock);
mac80211_hwsim_del_radio(data, wiphy_name(data->hw->wiphy),
info);
+ kfree(hwname);
return 0;
}
spin_unlock_bh(&hwsim_radio_lock);
+ kfree(hwname);
return -ENODEV;
}
int tx_failed;
};
-DECLARE_EWMA(rssi, 1024, 8)
+DECLARE_EWMA(rssi, 10, 8)
/*
* Antenna settings about the currently active link.
unsigned long flags;
bool found;
- new = kmalloc(sizeof(*entry), GFP_KERNEL);
+ new = kmalloc(sizeof(*entry), GFP_ATOMIC);
if (!new)
return;
#include "common.h"
#include <linux/kthread.h>
+#include <linux/sched/task.h>
#include <linux/ethtool.h>
#include <linux/rtnetlink.h>
#include <linux/if_vlan.h>
static void backend_disconnect(struct backend_info *be)
{
- if (be->vif) {
+ struct xenvif *vif = be->vif;
+
+ if (vif) {
unsigned int queue_index;
+ struct xenvif_queue *queues;
- xen_unregister_watchers(be->vif);
+ xen_unregister_watchers(vif);
#ifdef CONFIG_DEBUG_FS
- xenvif_debugfs_delif(be->vif);
+ xenvif_debugfs_delif(vif);
#endif /* CONFIG_DEBUG_FS */
- xenvif_disconnect_data(be->vif);
- for (queue_index = 0; queue_index < be->vif->num_queues; ++queue_index)
- xenvif_deinit_queue(&be->vif->queues[queue_index]);
+ xenvif_disconnect_data(vif);
+ for (queue_index = 0;
+ queue_index < vif->num_queues;
+ ++queue_index)
+ xenvif_deinit_queue(&vif->queues[queue_index]);
+
+ spin_lock(&vif->lock);
+ queues = vif->queues;
+ vif->num_queues = 0;
+ vif->queues = NULL;
+ spin_unlock(&vif->lock);
- spin_lock(&be->vif->lock);
- vfree(be->vif->queues);
- be->vif->num_queues = 0;
- be->vif->queues = NULL;
- spin_unlock(&be->vif->lock);
+ vfree(queues);
- xenvif_disconnect_ctrl(be->vif);
+ xenvif_disconnect_ctrl(vif);
}
}
struct device *create_namespace_pmem(struct nd_region *nd_region,
struct nd_namespace_label *nd_label)
{
+ u64 altcookie = nd_region_interleave_set_altcookie(nd_region);
u64 cookie = nd_region_interleave_set_cookie(nd_region);
struct nd_label_ent *label_ent;
struct nd_namespace_pmem *nspm;
if (__le64_to_cpu(nd_label->isetcookie) != cookie) {
dev_dbg(&nd_region->dev, "invalid cookie in label: %pUb\n",
nd_label->uuid);
- return ERR_PTR(-EAGAIN);
+ if (__le64_to_cpu(nd_label->isetcookie) != altcookie)
+ return ERR_PTR(-EAGAIN);
+
+ dev_dbg(&nd_region->dev, "valid altcookie in label: %pUb\n",
+ nd_label->uuid);
}
nspm = kzalloc(sizeof(*nspm), GFP_KERNEL);
res->name = dev_name(&nd_region->dev);
res->flags = IORESOURCE_MEM;
- for (i = 0; i < nd_region->ndr_mappings; i++)
- if (!has_uuid_at_pos(nd_region, nd_label->uuid, cookie, i))
- break;
+ for (i = 0; i < nd_region->ndr_mappings; i++) {
+ if (has_uuid_at_pos(nd_region, nd_label->uuid, cookie, i))
+ continue;
+ if (has_uuid_at_pos(nd_region, nd_label->uuid, altcookie, i))
+ continue;
+ break;
+ }
+
if (i < nd_region->ndr_mappings) {
struct nvdimm_drvdata *ndd = to_ndd(&nd_region->mapping[i]);
int nd_region_to_nstype(struct nd_region *nd_region);
int nd_region_register_namespaces(struct nd_region *nd_region, int *err);
u64 nd_region_interleave_set_cookie(struct nd_region *nd_region);
+u64 nd_region_interleave_set_altcookie(struct nd_region *nd_region);
void nvdimm_bus_lock(struct device *dev);
void nvdimm_bus_unlock(struct device *dev);
bool is_nvdimm_bus_locked(struct device *dev);
return 0;
}
+u64 nd_region_interleave_set_altcookie(struct nd_region *nd_region)
+{
+ struct nd_interleave_set *nd_set = nd_region->nd_set;
+
+ if (nd_set)
+ return nd_set->altcookie;
+ return 0;
+}
+
void nd_mapping_free_labels(struct nd_mapping *nd_mapping)
{
struct nd_label_ent *label_ent, *e;
}
EXPORT_SYMBOL_GPL(nvme_kill_queues);
+void nvme_unfreeze(struct nvme_ctrl *ctrl)
+{
+ struct nvme_ns *ns;
+
+ mutex_lock(&ctrl->namespaces_mutex);
+ list_for_each_entry(ns, &ctrl->namespaces, list)
+ blk_mq_unfreeze_queue(ns->queue);
+ mutex_unlock(&ctrl->namespaces_mutex);
+}
+EXPORT_SYMBOL_GPL(nvme_unfreeze);
+
+void nvme_wait_freeze_timeout(struct nvme_ctrl *ctrl, long timeout)
+{
+ struct nvme_ns *ns;
+
+ mutex_lock(&ctrl->namespaces_mutex);
+ list_for_each_entry(ns, &ctrl->namespaces, list) {
+ timeout = blk_mq_freeze_queue_wait_timeout(ns->queue, timeout);
+ if (timeout <= 0)
+ break;
+ }
+ mutex_unlock(&ctrl->namespaces_mutex);
+}
+EXPORT_SYMBOL_GPL(nvme_wait_freeze_timeout);
+
+void nvme_wait_freeze(struct nvme_ctrl *ctrl)
+{
+ struct nvme_ns *ns;
+
+ mutex_lock(&ctrl->namespaces_mutex);
+ list_for_each_entry(ns, &ctrl->namespaces, list)
+ blk_mq_freeze_queue_wait(ns->queue);
+ mutex_unlock(&ctrl->namespaces_mutex);
+}
+EXPORT_SYMBOL_GPL(nvme_wait_freeze);
+
+void nvme_start_freeze(struct nvme_ctrl *ctrl)
+{
+ struct nvme_ns *ns;
+
+ mutex_lock(&ctrl->namespaces_mutex);
+ list_for_each_entry(ns, &ctrl->namespaces, list)
+ blk_mq_freeze_queue_start(ns->queue);
+ mutex_unlock(&ctrl->namespaces_mutex);
+}
+EXPORT_SYMBOL_GPL(nvme_start_freeze);
+
void nvme_stop_queues(struct nvme_ctrl *ctrl)
{
struct nvme_ns *ns;
void nvme_stop_queues(struct nvme_ctrl *ctrl);
void nvme_start_queues(struct nvme_ctrl *ctrl);
void nvme_kill_queues(struct nvme_ctrl *ctrl);
+void nvme_unfreeze(struct nvme_ctrl *ctrl);
+void nvme_wait_freeze(struct nvme_ctrl *ctrl);
+void nvme_wait_freeze_timeout(struct nvme_ctrl *ctrl, long timeout);
+void nvme_start_freeze(struct nvme_ctrl *ctrl);
#define NVME_QID_ANY -1
struct request *nvme_alloc_request(struct request_queue *q,
}
static struct nvme_queue *nvme_alloc_queue(struct nvme_dev *dev, int qid,
- int depth)
+ int depth, int node)
{
- struct nvme_queue *nvmeq = kzalloc(sizeof(*nvmeq), GFP_KERNEL);
+ struct nvme_queue *nvmeq = kzalloc_node(sizeof(*nvmeq), GFP_KERNEL,
+ node);
if (!nvmeq)
return NULL;
nvmeq = dev->queues[0];
if (!nvmeq) {
- nvmeq = nvme_alloc_queue(dev, 0, NVME_AQ_DEPTH);
+ nvmeq = nvme_alloc_queue(dev, 0, NVME_AQ_DEPTH,
+ dev_to_node(dev->dev));
if (!nvmeq)
return -ENOMEM;
}
int ret = 0;
for (i = dev->queue_count; i <= dev->max_qid; i++) {
- if (!nvme_alloc_queue(dev, i, dev->q_depth)) {
+ /* vector == qid - 1, match nvme_create_queue */
+ if (!nvme_alloc_queue(dev, i, dev->q_depth,
+ pci_irq_get_node(to_pci_dev(dev->dev), i - 1))) {
ret = -ENOMEM;
break;
}
static void nvme_dev_disable(struct nvme_dev *dev, bool shutdown)
{
int i, queues;
- u32 csts = -1;
+ bool dead = true;
+ struct pci_dev *pdev = to_pci_dev(dev->dev);
del_timer_sync(&dev->watchdog_timer);
mutex_lock(&dev->shutdown_lock);
- if (pci_is_enabled(to_pci_dev(dev->dev))) {
- nvme_stop_queues(&dev->ctrl);
- csts = readl(dev->bar + NVME_REG_CSTS);
+ if (pci_is_enabled(pdev)) {
+ u32 csts = readl(dev->bar + NVME_REG_CSTS);
+
+ if (dev->ctrl.state == NVME_CTRL_LIVE)
+ nvme_start_freeze(&dev->ctrl);
+ dead = !!((csts & NVME_CSTS_CFS) || !(csts & NVME_CSTS_RDY) ||
+ pdev->error_state != pci_channel_io_normal);
}
+ /*
+ * Give the controller a chance to complete all entered requests if
+ * doing a safe shutdown.
+ */
+ if (!dead && shutdown)
+ nvme_wait_freeze_timeout(&dev->ctrl, NVME_IO_TIMEOUT);
+ nvme_stop_queues(&dev->ctrl);
+
queues = dev->online_queues - 1;
for (i = dev->queue_count - 1; i > 0; i--)
nvme_suspend_queue(dev->queues[i]);
- if (csts & NVME_CSTS_CFS || !(csts & NVME_CSTS_RDY)) {
+ if (dead) {
/* A device might become IO incapable very soon during
* probe, before the admin queue is configured. Thus,
* queue_count can be 0 here.
blk_mq_tagset_busy_iter(&dev->tagset, nvme_cancel_request, &dev->ctrl);
blk_mq_tagset_busy_iter(&dev->admin_tagset, nvme_cancel_request, &dev->ctrl);
+
+ /*
+ * The driver will not be starting up queues again if shutting down so
+ * must flush all entered requests to their failed completion to avoid
+ * deadlocking blk-mq hot-cpu notifier.
+ */
+ if (shutdown)
+ nvme_start_queues(&dev->ctrl);
mutex_unlock(&dev->shutdown_lock);
}
nvme_remove_namespaces(&dev->ctrl);
} else {
nvme_start_queues(&dev->ctrl);
+ nvme_wait_freeze(&dev->ctrl);
nvme_dev_add(dev);
+ nvme_unfreeze(&dev->ctrl);
}
if (!nvme_change_ctrl_state(&dev->ctrl, NVME_CTRL_LIVE)) {
*/
#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
#include <linux/module.h>
+#include <linux/rculist.h>
+
#include <generated/utsrelease.h>
#include <asm/unaligned.h>
#include "nvmet.h"
#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
#include <linux/module.h>
#include <linux/random.h>
+#include <linux/rculist.h>
+
#include "nvmet.h"
static struct nvmet_fabrics_ops *nvmet_transports[NVMF_TRTYPE_MAX];
#include <linux/fs.h>
#include <linux/oprofile.h>
#include <linux/sched.h>
+#include <linux/sched/mm.h>
+#include <linux/sched/task.h>
#include <linux/gfp.h>
#include "oprofile_stats.h"
#include <linux/oprofile.h>
#include <linux/errno.h>
+#include <asm/ptrace.h>
+
#include "event_buffer.h"
#include "cpu_buffer.h"
#include "buffer_sync.h"
#include <linux/vmalloc.h>
#include <linux/oprofile.h>
-#include <linux/sched.h>
+#include <linux/sched/signal.h>
#include <linux/capability.h>
#include <linux/dcookies.h>
#include <linux/fs.h>
ioc = kzalloc(sizeof(struct ioc), GFP_KERNEL);
if (ioc == NULL) {
printk(KERN_ERR MODULE_NAME ": memory allocation failure\n");
- return 1;
+ return -ENOMEM;
}
ioc->name = dev->id.hversion == U2_IOA_RUNWAY ? "U2" : "UTurn";
ioc->hw_path = dev->hw_path;
ioc->ioc_regs = ioremap_nocache(dev->hpa.start, 4096);
+ if (!ioc->ioc_regs) {
+ kfree(ioc);
+ return -ENOMEM;
+ }
ccio_ioc_init(ioc);
ccio_init_resources(ioc);
hppa_dma_ops = &ccio_ops;
* Wax ASIC also includes a PS/2 and RS-232 controller, but those are
* dealt with elsewhere; this file is concerned only with the EISA portions
* of Wax.
- *
- *
+ *
+ *
* HINT:
* -----
* To allow an ISA card to work properly in the EISA slot you need to
- * set an edge trigger level. This may be done on the palo command line
- * by adding the kernel parameter "eisa_irq_edge=n,n2,[...]]", with
+ * set an edge trigger level. This may be done on the palo command line
+ * by adding the kernel parameter "eisa_irq_edge=n,n2,[...]]", with
* n and n2 as the irq levels you want to use.
- *
- * Example: "eisa_irq_edge=10,11" allows ISA cards to operate at
+ *
+ * Example: "eisa_irq_edge=10,11" allows ISA cards to operate at
* irq levels 10 and 11.
*/
#include <asm/eisa_eeprom.h>
#if 0
-#define EISA_DBG(msg, arg... ) printk(KERN_DEBUG "eisa: " msg , ## arg )
+#define EISA_DBG(msg, arg...) printk(KERN_DEBUG "eisa: " msg, ## arg)
#else
-#define EISA_DBG(msg, arg... )
+#define EISA_DBG(msg, arg...)
#endif
#define SNAKES_EEPROM_BASE_ADDR 0xF0810400
void eisa_out16(unsigned short data, unsigned short port)
{
- if (EISA_bus)
+ if (EISA_bus)
gsc_writew(cpu_to_le16(data), eisa_permute(port));
}
static int slave_mask;
/* the trig level can be set with the
- * eisa_irq_edge=n,n,n commandline parameter
- * We should really read this from the EEPROM
- * in the furure.
+ * eisa_irq_edge=n,n,n commandline parameter
+ * We should really read this from the EEPROM
+ * in the furure.
*/
/* irq 13,8,2,1,0 must be edge */
static unsigned int eisa_irq_level __read_mostly; /* default to edge triggered */
unsigned int irq = d->irq;
unsigned long flags;
EISA_DBG("enable irq %d\n", irq);
-
+
spin_lock_irqsave(&eisa_irq_lock, flags);
if (irq & 8) {
slave_mask &= ~(1 << (irq&7));
{
int irq = gsc_readb(0xfc01f000); /* EISA supports 16 irqs */
unsigned long flags;
-
+
spin_lock_irqsave(&eisa_irq_lock, flags);
/* read IRR command */
eisa_out8(0x0a, 0x20);
EISA_DBG("irq IAR %02x 8259-1 irr %02x 8259-2 irr %02x\n",
irq, eisa_in8(0x20), eisa_in8(0xa0));
-
+
/* read ISR command */
eisa_out8(0x0a, 0x20);
eisa_out8(0x0a, 0xa0);
EISA_DBG("irq 8259-1 isr %02x imr %02x 8259-2 isr %02x imr %02x\n",
eisa_in8(0x20), eisa_in8(0x21), eisa_in8(0xa0), eisa_in8(0xa1));
-
+
irq &= 0xf;
-
+
/* mask irq and write eoi */
if (irq & 8) {
slave_mask |= (1 << (irq&7));
eisa_out8(slave_mask, 0xa1);
eisa_out8(0x60 | (irq&7),0xa0);/* 'Specific EOI' to slave */
- eisa_out8(0x62,0x20); /* 'Specific EOI' to master-IRQ2 */
-
+ eisa_out8(0x62, 0x20); /* 'Specific EOI' to master-IRQ2 */
+
} else {
master_mask |= (1 << (irq&7));
eisa_out8(master_mask, 0x21);
- eisa_out8(0x60|irq,0x20); /* 'Specific EOI' to master */
+ eisa_out8(0x60|irq, 0x20); /* 'Specific EOI' to master */
}
spin_unlock_irqrestore(&eisa_irq_lock, flags);
generic_handle_irq(irq);
-
+
spin_lock_irqsave(&eisa_irq_lock, flags);
/* unmask */
if (irq & 8) {
static void init_eisa_pic(void)
{
unsigned long flags;
-
+
spin_lock_irqsave(&eisa_irq_lock, flags);
eisa_out8(0xff, 0x21); /* mask during init */
eisa_out8(0xff, 0xa1); /* mask during init */
-
+
/* master pic */
- eisa_out8(0x11,0x20); /* ICW1 */
- eisa_out8(0x00,0x21); /* ICW2 */
- eisa_out8(0x04,0x21); /* ICW3 */
- eisa_out8(0x01,0x21); /* ICW4 */
- eisa_out8(0x40,0x20); /* OCW2 */
-
+ eisa_out8(0x11, 0x20); /* ICW1 */
+ eisa_out8(0x00, 0x21); /* ICW2 */
+ eisa_out8(0x04, 0x21); /* ICW3 */
+ eisa_out8(0x01, 0x21); /* ICW4 */
+ eisa_out8(0x40, 0x20); /* OCW2 */
+
/* slave pic */
- eisa_out8(0x11,0xa0); /* ICW1 */
- eisa_out8(0x08,0xa1); /* ICW2 */
- eisa_out8(0x02,0xa1); /* ICW3 */
- eisa_out8(0x01,0xa1); /* ICW4 */
- eisa_out8(0x40,0xa0); /* OCW2 */
-
+ eisa_out8(0x11, 0xa0); /* ICW1 */
+ eisa_out8(0x08, 0xa1); /* ICW2 */
+ eisa_out8(0x02, 0xa1); /* ICW3 */
+ eisa_out8(0x01, 0xa1); /* ICW4 */
+ eisa_out8(0x40, 0xa0); /* OCW2 */
+
udelay(100);
-
- slave_mask = 0xff;
- master_mask = 0xfb;
+
+ slave_mask = 0xff;
+ master_mask = 0xfb;
eisa_out8(slave_mask, 0xa1); /* OCW1 */
eisa_out8(master_mask, 0x21); /* OCW1 */
-
+
/* setup trig level */
EISA_DBG("EISA edge/level %04x\n", eisa_irq_level);
-
+
eisa_out8(eisa_irq_level&0xff, 0x4d0); /* Set all irq's to edge */
- eisa_out8((eisa_irq_level >> 8) & 0xff, 0x4d1);
-
+ eisa_out8((eisa_irq_level >> 8) & 0xff, 0x4d1);
+
EISA_DBG("pic0 mask %02x\n", eisa_in8(0x21));
EISA_DBG("pic1 mask %02x\n", eisa_in8(0xa1));
EISA_DBG("pic0 edge/level %02x\n", eisa_in8(0x4d0));
EISA_DBG("pic1 edge/level %02x\n", eisa_in8(0x4d1));
-
+
spin_unlock_irqrestore(&eisa_irq_lock, flags);
}
char *name = is_mongoose(dev) ? "Mongoose" : "Wax";
- printk(KERN_INFO "%s EISA Adapter found at 0x%08lx\n",
+ printk(KERN_INFO "%s EISA Adapter found at 0x%08lx\n",
name, (unsigned long)dev->hpa.start);
eisa_dev.hba.dev = dev;
result = request_irq(dev->irq, eisa_irq, IRQF_SHARED, "EISA", &eisa_dev);
if (result) {
printk(KERN_ERR "EISA: request_irq failed!\n");
- return result;
+ goto error_release;
}
-
+
/* Reserve IRQ2 */
setup_irq(2, &irq2_action);
for (i = 0; i < 16; i++) {
irq_set_chip_and_handler(i, &eisa_interrupt_type,
handle_simple_irq);
}
-
+
EISA_bus = 1;
if (dev->num_addrs) {
}
}
eisa_eeprom_addr = ioremap_nocache(eisa_dev.eeprom_addr, HPEE_MAX_LENGTH);
+ if (!eisa_eeprom_addr) {
+ result = -ENOMEM;
+ printk(KERN_ERR "EISA: ioremap_nocache failed!\n");
+ goto error_free_irq;
+ }
result = eisa_enumerator(eisa_dev.eeprom_addr, &eisa_dev.hba.io_space,
&eisa_dev.hba.lmmio_space);
init_eisa_pic();
eisa_dev.root.dma_mask = 0xffffffff; /* wild guess */
if (eisa_root_register (&eisa_dev.root)) {
printk(KERN_ERR "EISA: Failed to register EISA root\n");
- return -1;
+ result = -ENOMEM;
+ goto error_iounmap;
}
}
-
+
return 0;
+
+error_iounmap:
+ iounmap(eisa_eeprom_addr);
+error_free_irq:
+ free_irq(dev->irq, &eisa_dev);
+error_release:
+ release_resource(&eisa_dev.hba.io_space);
+ return result;
}
static const struct parisc_device_id eisa_tbl[] = {
{
if (eisa_irq_configured& (1<<num)) {
printk(KERN_WARNING
- "IRQ %d polarity configured twice (last to level)\n",
+ "IRQ %d polarity configured twice (last to level)\n",
num);
}
eisa_irq_level |= (1<<num); /* set the corresponding bit */
void eisa_make_irq_edge(int num)
{
if (eisa_irq_configured& (1<<num)) {
- printk(KERN_WARNING
+ printk(KERN_WARNING
"IRQ %d polarity configured twice (last to edge)\n",
num);
}
EISA_DBG("IRQ setup\n");
while (cur != NULL) {
char *pe;
-
+
val = (int) simple_strtoul(cur, &pe, 0);
if (val > 15 || val < 0) {
printk(KERN_ERR "eisa: EISA irq value are 0-15\n");
continue;
}
- if (val == 2) {
+ if (val == 2) {
val = 9;
}
eisa_make_irq_edge(val); /* clear the corresponding bit */
EISA_DBG("setting IRQ %d to edge-triggered mode\n", val);
-
+
if ((cur = strchr(cur, ','))) {
cur++;
} else {
#include <linux/kernel.h>
#include <linux/notifier.h>
#include <linux/reboot.h>
-#include <linux/sched.h>
+#include <linux/sched/signal.h>
#include <linux/kthread.h>
#include <linux/pm.h>
#include <linux/parport.h>
#include <linux/delay.h>
#include <linux/slab.h>
-#include <linux/sched.h>
+#include <linux/sched/signal.h>
#include <asm/current.h>
#include <linux/uaccess.h>
#include <linux/kernel.h>
#include <linux/interrupt.h>
#include <linux/timer.h>
-#include <linux/sched.h>
+#include <linux/sched/signal.h>
#undef DEBUG /* undef me for production */
#include <linux/module.h>
#include <linux/parport.h>
#include <linux/delay.h>
-#include <linux/sched.h>
+#include <linux/sched/signal.h>
#include <linux/uaccess.h>
#undef DEBUG /* undef me for production */
#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/parport.h>
-#include <linux/sched.h>
+#include <linux/sched/signal.h>
#include <linux/slab.h>
#include <linux/spinlock.h>
#include <linux/stddef.h>
#include <linux/module.h>
#include <linux/init.h>
-#include <linux/sched.h>
+#include <linux/sched/signal.h>
#include <linux/delay.h>
#include <linux/errno.h>
#include <linux/interrupt.h>
#include <linux/ioport.h>
#include <linux/kernel.h>
#include <linux/slab.h>
-#include <linux/sched.h>
+#include <linux/sched/signal.h>
#include <linux/kmod.h>
#include <linux/device.h>
#include <linux/delay.h>
#include <linux/pci.h>
#include <linux/module.h>
-#include <linux/sched.h>
+#include <linux/sched/signal.h>
#include <linux/slab.h>
#include <linux/ioport.h>
#include <linux/wait.h>
struct device *dev = pci->dev;
struct resource *res;
- /* If using the PHY framework, doesn't need to get other resource */
- if (ep->using_phy)
- return 0;
-
ep->mem_res = devm_kzalloc(dev, sizeof(*ep->mem_res), GFP_KERNEL);
if (!ep->mem_res)
return -ENOMEM;
if (IS_ERR(ep->mem_res->elbi_base))
return PTR_ERR(ep->mem_res->elbi_base);
+ /* If using the PHY framework, doesn't need to get other resource */
+ if (ep->using_phy)
+ return 0;
+
res = platform_get_resource(pdev, IORESOURCE_MEM, 1);
ep->mem_res->phy_base = devm_ioremap_resource(dev, res);
if (IS_ERR(ep->mem_res->phy_base))
pci->dev = dev;
pci->ops = &dw_pcie_ops;
+ ep->pci = pci;
ep->ops = (const struct exynos_pcie_ops *)
of_device_get_match_data(dev);
pci->dev = dev;
pci->ops = &dw_pcie_ops;
+ imx6_pcie->pci = pci;
imx6_pcie->variant =
(enum imx6_pcie_variants)of_device_get_match_data(dev);
pci->dev = dev;
pci->ops = &dw_pcie_ops;
+ ks_pcie->pci = pci;
+
/* initialize SerDes Phy if present */
phy = devm_phy_get(dev, "pcie-phy");
if (PTR_ERR_OR_ZERO(phy) == -EPROBE_DEFER)
pci->dev = dev;
pci->ops = pcie->drvdata->dw_pcie_ops;
+ pcie->pci = pci;
+
dbi_base = platform_get_resource_byname(pdev, IORESOURCE_MEM, "regs");
pci->dbi_base = devm_ioremap_resource(dev, dbi_base);
if (IS_ERR(pci->dbi_base))
pci->dev = dev;
pci->ops = &dw_pcie_ops;
+ pcie->pci = pci;
+
pcie->clk = devm_clk_get(dev, NULL);
if (IS_ERR(pcie->clk))
return PTR_ERR(pcie->clk);
pci->dev = dev;
+ artpec6_pcie->pci = pci;
+
dbi_base = platform_get_resource_byname(pdev, IORESOURCE_MEM, "dbi");
pci->dbi_base = devm_ioremap_resource(dev, dbi_base);
if (IS_ERR(pci->dbi_base))
pci->dev = dev;
+ dw_plat_pcie->pci = pci;
+
res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
pci->dbi_base = devm_ioremap_resource(dev, res);
if (IS_ERR(pci->dbi_base))
driver = dev->driver;
+ hisi_pcie->pci = pci;
+
hisi_pcie->soc_ops = of_device_get_match_data(dev);
hisi_pcie->subctrl =
pci->ops = &dw_pcie_ops;
pp = &pci->pp;
+ pcie->pci = pci;
+
pcie->ops = (struct qcom_pcie_ops *)of_device_get_match_data(dev);
pcie->reset = devm_gpiod_get_optional(dev, "perst", GPIOD_OUT_LOW);
pci->dev = dev;
pci->ops = &dw_pcie_ops;
+ spear13xx_pcie->pci = pci;
+
spear13xx_pcie->phy = devm_phy_get(dev, "pcie-phy");
if (IS_ERR(spear13xx_pcie->phy)) {
ret = PTR_ERR(spear13xx_pcie->phy);
#define TLP_WRITE_TAG 0x10
#define RP_DEVFN 0
#define TLP_REQ_ID(bus, devfn) (((bus) << 8) | (devfn))
-#define TLP_CFG_DW0(pcie, bus) \
+#define TLP_CFGRD_DW0(pcie, bus) \
((((bus == pcie->root_bus_nr) ? TLP_FMTTYPE_CFGRD0 \
: TLP_FMTTYPE_CFGRD1) << 24) | \
TLP_PAYLOAD_SIZE)
+#define TLP_CFGWR_DW0(pcie, bus) \
+ ((((bus == pcie->root_bus_nr) ? TLP_FMTTYPE_CFGWR0 \
+ : TLP_FMTTYPE_CFGWR1) << 24) | \
+ TLP_PAYLOAD_SIZE)
#define TLP_CFG_DW1(pcie, tag, be) \
(((TLP_REQ_ID(pcie->root_bus_nr, RP_DEVFN)) << 16) | (tag << 8) | (be))
#define TLP_CFG_DW2(bus, devfn, offset) \
{
u32 headers[TLP_HDR_SIZE];
- headers[0] = TLP_CFG_DW0(pcie, bus);
+ headers[0] = TLP_CFGRD_DW0(pcie, bus);
headers[1] = TLP_CFG_DW1(pcie, TLP_READ_TAG, byte_en);
headers[2] = TLP_CFG_DW2(bus, devfn, where);
u32 headers[TLP_HDR_SIZE];
int ret;
- headers[0] = TLP_CFG_DW0(pcie, bus);
+ headers[0] = TLP_CFGWR_DW0(pcie, bus);
headers[1] = TLP_CFG_DW1(pcie, TLP_WRITE_TAG, byte_en);
headers[2] = TLP_CFG_DW2(bus, devfn, where);
#include <linux/module.h>
#include <linux/kernel.h>
+#include <linux/sched/signal.h>
#include <linux/slab.h>
#include <linux/pci.h>
#include <linux/pci_hotplug.h>
#include <asm/io.h> /* for read? and write? functions */
#include <linux/delay.h> /* for delays */
#include <linux/mutex.h>
-#include <linux/sched.h> /* for signal_pending() */
+#include <linux/sched/signal.h> /* for signal_pending() */
#define MY_NAME "cpqphp"
#include <linux/pci.h>
#include <linux/pci_hotplug.h>
#include <linux/delay.h>
-#include <linux/sched.h> /* signal_pending() */
+#include <linux/sched/signal.h> /* signal_pending() */
#include <linux/pcieport_if.h>
#include <linux/mutex.h>
#include <linux/workqueue.h>
#include <linux/pci.h>
#include <linux/pci_hotplug.h>
#include <linux/delay.h>
-#include <linux/sched.h> /* signal_pending(), struct timer_list */
+#include <linux/sched/signal.h> /* signal_pending(), struct timer_list */
#include <linux/mutex.h>
#include <linux/workqueue.h>
}
EXPORT_SYMBOL(pci_irq_get_affinity);
+/**
+ * pci_irq_get_node - return the numa node of a particular msi vector
+ * @pdev: PCI device to operate on
+ * @vec: device-relative interrupt vector index (0-based).
+ */
+int pci_irq_get_node(struct pci_dev *pdev, int vec)
+{
+ const struct cpumask *mask;
+
+ mask = pci_irq_get_affinity(pdev, vec);
+ if (mask)
+ return local_memory_node(cpu_to_node(cpumask_first(mask)));
+ return dev_to_node(&pdev->dev);
+}
+EXPORT_SYMBOL(pci_irq_get_node);
+
struct pci_dev *msi_desc_to_pci_dev(struct msi_desc *desc)
{
return to_pci_dev(desc->dev);
static void pcie_aspm_cap_init(struct pcie_link_state *link, int blacklist)
{
- struct pci_dev *child, *parent = link->pdev;
+ struct pci_dev *child = link->downstream, *parent = link->pdev;
struct pci_bus *linkbus = parent->subordinate;
struct aspm_register_info upreg, dwreg;
/* Get upstream/downstream components' register state */
pcie_get_aspm_reg(parent, &upreg);
- child = pci_function_0(linkbus);
pcie_get_aspm_reg(child, &dwreg);
- link->downstream = child;
/*
* If ASPM not supported, don't mess with the clocks and link,
INIT_LIST_HEAD(&link->children);
INIT_LIST_HEAD(&link->link);
link->pdev = pdev;
+ link->downstream = pci_function_0(pdev->subordinate);
/*
* Root Ports and PCI/PCI-X to PCIe Bridges are roots of PCIe
DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_LSI_LOGIC, 0x005f, quirk_blacklist_vpd);
DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_ATTANSIC, PCI_ANY_ID,
quirk_blacklist_vpd);
+DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_QLOGIC, 0x2261, quirk_blacklist_vpd);
/*
* For Broadcom 5706, 5708, 5709 rev. A nics, any read beyond the
#include <linux/perf/arm_pmu.h>
#include <linux/platform_device.h>
#include <linux/slab.h>
+#include <linux/sched/clock.h>
#include <linux/spinlock.h>
#include <linux/irq.h>
#include <linux/irqdesc.h>
return 0;
}
+static int msm_gpio_get_direction(struct gpio_chip *chip, unsigned int offset)
+{
+ struct msm_pinctrl *pctrl = gpiochip_get_data(chip);
+ const struct msm_pingroup *g;
+ u32 val;
+
+ g = &pctrl->soc->groups[offset];
+
+ val = readl(pctrl->regs + g->ctl_reg);
+
+ /* 0 = output, 1 = input */
+ return val & BIT(g->oe_bit) ? 0 : 1;
+}
+
static int msm_gpio_get(struct gpio_chip *chip, unsigned offset)
{
const struct msm_pingroup *g;
static struct gpio_chip msm_gpio_template = {
.direction_input = msm_gpio_direction_input,
.direction_output = msm_gpio_direction_output,
+ .get_direction = msm_gpio_get_direction,
.get = msm_gpio_get,
.set = msm_gpio_set,
.request = gpiochip_generic_request,
UNIPHIER_PINCTRL_PIN(140, "AO1D0", 140,
140, UNIPHIER_PIN_DRV_1BIT,
140, UNIPHIER_PIN_PULL_DOWN),
- UNIPHIER_PINCTRL_PIN(141, "TCON0", 141,
+ UNIPHIER_PINCTRL_PIN(141, "AO1D1", 141,
141, UNIPHIER_PIN_DRV_1BIT,
141, UNIPHIER_PIN_PULL_DOWN),
- UNIPHIER_PINCTRL_PIN(142, "TCON1", 142,
+ UNIPHIER_PINCTRL_PIN(142, "AO1D2", 142,
142, UNIPHIER_PIN_DRV_1BIT,
142, UNIPHIER_PIN_PULL_DOWN),
- UNIPHIER_PINCTRL_PIN(143, "TCON2", 143,
+ UNIPHIER_PINCTRL_PIN(143, "XIRQ9", 143,
143, UNIPHIER_PIN_DRV_1BIT,
143, UNIPHIER_PIN_PULL_DOWN),
- UNIPHIER_PINCTRL_PIN(144, "TCON3", 144,
+ UNIPHIER_PINCTRL_PIN(144, "XIRQ10", 144,
144, UNIPHIER_PIN_DRV_1BIT,
144, UNIPHIER_PIN_PULL_DOWN),
- UNIPHIER_PINCTRL_PIN(145, "TCON4", 145,
+ UNIPHIER_PINCTRL_PIN(145, "XIRQ11", 145,
145, UNIPHIER_PIN_DRV_1BIT,
145, UNIPHIER_PIN_PULL_DOWN),
- UNIPHIER_PINCTRL_PIN(146, "TCON5", 146,
+ UNIPHIER_PINCTRL_PIN(146, "XIRQ13", 146,
146, UNIPHIER_PIN_DRV_1BIT,
146, UNIPHIER_PIN_PULL_DOWN),
UNIPHIER_PINCTRL_PIN(147, "PWMA", 147,
#include <linux/module.h>
#include <linux/pci.h>
#include <linux/sched.h>
+#include <linux/sched/loadavg.h>
#include <linux/seq_file.h>
#include <linux/string.h>
#include <linux/tick.h>
#include <linux/module.h>
#include <linux/workqueue.h>
#include <linux/reboot.h>
+#include <linux/sched/signal.h>
#include <asm/firmware.h>
#include <asm/lv1call.h>
static int rproc_virtio_find_vqs(struct virtio_device *vdev, unsigned int nvqs,
struct virtqueue *vqs[],
vq_callback_t *callbacks[],
- const char * const names[])
+ const char * const names[],
+ struct irq_affinity *desc)
{
int i, ret;
init_waitqueue_head(&vrp->sendq);
/* We expect two virtqueues, rx and tx (and in this order) */
- err = vdev->config->find_vqs(vdev, 2, vqs, vq_cbs, names);
+ err = vdev->config->find_vqs(vdev, 2, vqs, vq_cbs, names, NULL);
if (err)
goto free_vrp;
#include <linux/module.h>
#include <linux/rtc.h>
-#include <linux/sched.h>
+#include <linux/sched/signal.h>
#include "rtc-core.h"
static dev_t rtc_devt;
#include <linux/init.h>
#include <linux/interrupt.h>
#include <linux/compat.h>
+#include <linux/sched/signal.h>
#include <linux/module.h>
#include <linux/list.h>
#include <linux/slab.h>
*/
#include <linux/module.h>
-#include <linux/sched.h>
+#include <linux/sched/signal.h>
#include <linux/slab.h>
#include <linux/sysrq.h>
#include <asm/chpid.h>
#include <asm/airq.h>
#include <asm/isc.h>
-#include <linux/cputime.h>
+#include <linux/sched/cputime.h>
#include <asm/fcx.h>
#include <asm/nmi.h>
#include <asm/crw.h>
#include <linux/delay.h>
#include <linux/timer.h>
#include <linux/kernel_stat.h>
+#include <linux/sched/signal.h>
#include <asm/ccwdev.h>
#include <asm/cio.h>
#include <linux/slab.h>
#include <linux/kernel_stat.h>
#include <linux/atomic.h>
+#include <linux/rculist.h>
+
#include <asm/debug.h>
#include <asm/qdio.h>
#include <asm/airq.h>
static int kvm_find_vqs(struct virtio_device *vdev, unsigned nvqs,
struct virtqueue *vqs[],
vq_callback_t *callbacks[],
- const char * const names[])
+ const char * const names[],
+ struct irq_affinity *desc)
{
struct kvm_device *kdev = to_kvmdev(vdev);
int i;
static int virtio_ccw_find_vqs(struct virtio_device *vdev, unsigned nvqs,
struct virtqueue *vqs[],
vq_callback_t *callbacks[],
- const char * const names[])
+ const char * const names[],
+ struct irq_affinity *desc)
{
struct virtio_ccw_device *vcdev = to_vc_device(vdev);
unsigned long *indicatorp = NULL;
source "drivers/scsi/qla2xxx/Kconfig"
source "drivers/scsi/qla4xxx/Kconfig"
source "drivers/scsi/qedi/Kconfig"
+source "drivers/scsi/qedf/Kconfig"
config SCSI_LPFC
tristate "Emulex LightPulse Fibre Channel Support"
depends on PCI && SCSI
depends on SCSI_FC_ATTRS
+ depends on NVME_FC && NVME_TARGET_FC
select CRC_T10DIF
help
This lpfc driver supports the Emulex LightPulse
config MAC_SCSI
tristate "Macintosh NCR5380 SCSI"
- depends on MAC && SCSI=y
+ depends on MAC && SCSI
select SCSI_SPI_ATTRS
help
This is the NCR 5380 SCSI controller included on most of the 68030
obj-$(CONFIG_FCOE_FNIC) += fnic/
obj-$(CONFIG_SCSI_SNIC) += snic/
obj-$(CONFIG_SCSI_BNX2X_FCOE) += libfc/ fcoe/ bnx2fc/
+obj-$(CONFIG_QEDF) += qedf/
obj-$(CONFIG_ISCSI_TCP) += libiscsi.o libiscsi_tcp.o iscsi_tcp.o
obj-$(CONFIG_INFINIBAND_ISER) += libiscsi.o
obj-$(CONFIG_ISCSI_BOOT_SYSFS) += iscsi_boot_sysfs.o
"deregistering them. This is typically adjusted for heavily burdened"
" systems.");
+int aac_fib_dump;
+module_param(aac_fib_dump, int, 0644);
+MODULE_PARM_DESC(aac_fib_dump, "Dump controller fibs prior to IOP_RESET 0=off, 1=on");
+
int numacb = -1;
module_param(numacb, int, S_IRUGO|S_IWUSR);
MODULE_PARM_DESC(numacb, "Request a limit to the number of adapter control"
MODULE_PARM_DESC(update_interval, "Interval in seconds between time sync"
" updates issued to adapter.");
-int check_interval = 24 * 60 * 60;
+int check_interval = 60;
module_param(check_interval, int, S_IRUGO|S_IWUSR);
MODULE_PARM_DESC(check_interval, "Interval in seconds between adapter health"
" checks.");
if (status >= 0) {
dresp = (struct aac_get_container_count_resp *)fib_data(fibptr);
maximum_num_containers = le32_to_cpu(dresp->ContainerSwitchEntries);
- if (fibptr->dev->supplement_adapter_info.SupportedOptions2 &
+ if (fibptr->dev->supplement_adapter_info.supported_options2 &
AAC_OPTION_SUPPORTED_240_VOLUMES) {
maximum_num_containers =
le32_to_cpu(dresp->MaxSimpleVolumes);
fsa_dev_ptr = fibptr->dev->fsa_dev;
if (fsa_dev_ptr) {
struct aac_mount * dresp = (struct aac_mount *) fib_data(fibptr);
+ __le32 sup_options2;
+
fsa_dev_ptr += scmd_id(scsicmd);
+ sup_options2 =
+ fibptr->dev->supplement_adapter_info.supported_options2;
if ((le32_to_cpu(dresp->status) == ST_OK) &&
(le32_to_cpu(dresp->mnt[0].vol) != CT_NONE) &&
(le32_to_cpu(dresp->mnt[0].state) != FSCS_HIDDEN)) {
- if (!(fibptr->dev->supplement_adapter_info.SupportedOptions2 &
- AAC_OPTION_VARIABLE_BLOCK_SIZE)) {
+ if (!(sup_options2 & AAC_OPTION_VARIABLE_BLOCK_SIZE)) {
dresp->mnt[0].fileinfo.bdevinfo.block_size = 0x200;
fsa_dev_ptr->block_size = 0x200;
} else {
int status;
dresp = (struct aac_mount *) fib_data(fibptr);
- if (!(fibptr->dev->supplement_adapter_info.SupportedOptions2 &
+ if (!(fibptr->dev->supplement_adapter_info.supported_options2 &
AAC_OPTION_VARIABLE_BLOCK_SIZE))
dresp->mnt[0].capacityhigh = 0;
if ((le32_to_cpu(dresp->status) != ST_OK) ||
dinfo = (struct aac_query_mount *)fib_data(fibptr);
- if (fibptr->dev->supplement_adapter_info.SupportedOptions2 &
+ if (fibptr->dev->supplement_adapter_info.supported_options2 &
AAC_OPTION_VARIABLE_BLOCK_SIZE)
dinfo->command = cpu_to_le32(VM_NameServeAllBlk);
else
dinfo = (struct aac_query_mount *)fib_data(fibptr);
- if (fibptr->dev->supplement_adapter_info.SupportedOptions2 &
+ if (fibptr->dev->supplement_adapter_info.supported_options2 &
AAC_OPTION_VARIABLE_BLOCK_SIZE)
dinfo->command = cpu_to_le32(VM_NameServeAllBlk);
else
static void setinqstr(struct aac_dev *dev, void *data, int tindex)
{
struct scsi_inq *str;
+ struct aac_supplement_adapter_info *sup_adap_info;
+ sup_adap_info = &dev->supplement_adapter_info;
str = (struct scsi_inq *)(data); /* cast data to scsi inq block */
memset(str, ' ', sizeof(*str));
- if (dev->supplement_adapter_info.AdapterTypeText[0]) {
- char * cp = dev->supplement_adapter_info.AdapterTypeText;
+ if (sup_adap_info->adapter_type_text[0]) {
+ char *cp = sup_adap_info->adapter_type_text;
int c;
if ((cp[0] == 'A') && (cp[1] == 'O') && (cp[2] == 'C'))
inqstrcpy("SMC", str->vid);
++cp;
c = *cp;
*cp = '\0';
- inqstrcpy (dev->supplement_adapter_info.AdapterTypeText,
- str->vid);
+ inqstrcpy(sup_adap_info->adapter_type_text, str->vid);
*cp = c;
while (*cp && *cp != ' ')
++cp;
if (!identify_resp)
goto fib_free_ptr;
- vbus = (u32)le16_to_cpu(dev->supplement_adapter_info.VirtDeviceBus);
- vid = (u32)le16_to_cpu(dev->supplement_adapter_info.VirtDeviceTarget);
+ vbus = (u32)le16_to_cpu(dev->supplement_adapter_info.virt_device_bus);
+ vid = (u32)le16_to_cpu(dev->supplement_adapter_info.virt_device_target);
aac_fib_init(fibptr);
}
vbus = (u32) le16_to_cpu(
- dev->supplement_adapter_info.VirtDeviceBus);
+ dev->supplement_adapter_info.virt_device_bus);
vid = (u32) le16_to_cpu(
- dev->supplement_adapter_info.VirtDeviceTarget);
+ dev->supplement_adapter_info.virt_device_target);
aac_fib_init(fibptr);
}
memcpy(&dev->adapter_info, info, sizeof(*info));
- dev->supplement_adapter_info.VirtDeviceBus = 0xffff;
+ dev->supplement_adapter_info.virt_device_bus = 0xffff;
if (dev->adapter_info.options & AAC_OPT_SUPPLEMENT_ADAPTER_INFO) {
struct aac_supplement_adapter_info * sinfo;
}
if (!dev->sync_mode && dev->sa_firmware &&
- dev->supplement_adapter_info.VirtDeviceBus != 0xffff) {
+ dev->supplement_adapter_info.virt_device_bus != 0xffff) {
/* Thor SA Firmware -> CISS_REPORT_PHYSICAL_LUNS */
rcode = aac_report_phys_luns(dev, fibptr, AAC_INIT);
}
(tmp>>16)&0xff,
tmp&0xff,
le32_to_cpu(dev->adapter_info.kernelbuild),
- (int)sizeof(dev->supplement_adapter_info.BuildDate),
- dev->supplement_adapter_info.BuildDate);
+ (int)sizeof(dev->supplement_adapter_info.build_date),
+ dev->supplement_adapter_info.build_date);
tmp = le32_to_cpu(dev->adapter_info.monitorrev);
printk(KERN_INFO "%s%d: monitor %d.%d-%d[%d]\n",
dev->name, dev->id,
shost_to_class(dev->scsi_host_ptr), buffer))
printk(KERN_INFO "%s%d: serial %s",
dev->name, dev->id, buffer);
- if (dev->supplement_adapter_info.VpdInfo.Tsid[0]) {
+ if (dev->supplement_adapter_info.vpd_info.tsid[0]) {
printk(KERN_INFO "%s%d: TSID %.*s\n",
dev->name, dev->id,
- (int)sizeof(dev->supplement_adapter_info.VpdInfo.Tsid),
- dev->supplement_adapter_info.VpdInfo.Tsid);
+ (int)sizeof(dev->supplement_adapter_info
+ .vpd_info.tsid),
+ dev->supplement_adapter_info.vpd_info.tsid);
}
if (!aac_check_reset || ((aac_check_reset == 1) &&
- (dev->supplement_adapter_info.SupportedOptions2 &
+ (dev->supplement_adapter_info.supported_options2 &
AAC_OPTION_IGNORE_RESET))) {
printk(KERN_INFO "%s%d: Reset Adapter Ignored\n",
dev->name, dev->id);
}
dev->cache_protected = 0;
- dev->jbod = ((dev->supplement_adapter_info.FeatureBits &
+ dev->jbod = ((dev->supplement_adapter_info.feature_bits &
AAC_FEATURE_JBOD) != 0);
dev->nondasd_support = 0;
dev->raid_scsi_mode = 0;
struct scsi_device *sdev = scsicmd->device;
struct aac_dev *aac = (struct aac_dev *)sdev->host->hostdata;
- if (!(aac->supplement_adapter_info.SupportedOptions2 &
+ if (!(aac->supplement_adapter_info.supported_options2 &
AAC_OPTION_POWER_MANAGEMENT)) {
scsicmd->result = DID_OK << 16 | COMMAND_COMPLETE << 8 |
SAM_STAT_GOOD;
#define PMC_GLOBAL_INT_BIT0 0x00000001
#ifndef AAC_DRIVER_BUILD
-# define AAC_DRIVER_BUILD 50740
+# define AAC_DRIVER_BUILD 50792
# define AAC_DRIVER_BRANCH "-custom"
#endif
#define MAXIMUM_NUM_CONTAINERS 32
struct aac_supplement_adapter_info
{
- u8 AdapterTypeText[17+1];
- u8 Pad[2];
- __le32 FlashMemoryByteSize;
- __le32 FlashImageId;
- __le32 MaxNumberPorts;
- __le32 Version;
- __le32 FeatureBits;
- u8 SlotNumber;
- u8 ReservedPad0[3];
- u8 BuildDate[12];
- __le32 CurrentNumberPorts;
+ u8 adapter_type_text[17+1];
+ u8 pad[2];
+ __le32 flash_memory_byte_size;
+ __le32 flash_image_id;
+ __le32 max_number_ports;
+ __le32 version;
+ __le32 feature_bits;
+ u8 slot_number;
+ u8 reserved_pad0[3];
+ u8 build_date[12];
+ __le32 current_number_ports;
struct {
- u8 AssemblyPn[8];
- u8 FruPn[8];
- u8 BatteryFruPn[8];
- u8 EcVersionString[8];
- u8 Tsid[12];
- } VpdInfo;
- __le32 FlashFirmwareRevision;
- __le32 FlashFirmwareBuild;
- __le32 RaidTypeMorphOptions;
- __le32 FlashFirmwareBootRevision;
- __le32 FlashFirmwareBootBuild;
- u8 MfgPcbaSerialNo[12];
- u8 MfgWWNName[8];
- __le32 SupportedOptions2;
- __le32 StructExpansion;
+ u8 assembly_pn[8];
+ u8 fru_pn[8];
+ u8 battery_fru_pn[8];
+ u8 ec_version_string[8];
+ u8 tsid[12];
+ } vpd_info;
+ __le32 flash_firmware_revision;
+ __le32 flash_firmware_build;
+ __le32 raid_type_morph_options;
+ __le32 flash_firmware_boot_revision;
+ __le32 flash_firmware_boot_build;
+ u8 mfg_pcba_serial_no[12];
+ u8 mfg_wwn_name[8];
+ __le32 supported_options2;
+ __le32 struct_expansion;
/* StructExpansion == 1 */
- __le32 FeatureBits3;
- __le32 SupportedPerformanceModes;
- u8 HostBusType; /* uses HOST_BUS_TYPE_xxx defines */
- u8 HostBusWidth; /* actual width in bits or links */
- u16 HostBusSpeed; /* actual bus speed/link rate in MHz */
- u8 MaxRRCDrives; /* max. number of ITP-RRC drives/pool */
- u8 MaxDiskXtasks; /* max. possible num of DiskX Tasks */
-
- u8 CpldVerLoaded;
- u8 CpldVerInFlash;
-
- __le64 MaxRRCCapacity;
- __le32 CompiledMaxHistLogLevel;
- u8 CustomBoardName[12];
- u16 SupportedCntlrMode; /* identify supported controller mode */
- u16 ReservedForFuture16;
- __le32 SupportedOptions3; /* reserved for future options */
-
- __le16 VirtDeviceBus; /* virt. SCSI device for Thor */
- __le16 VirtDeviceTarget;
- __le16 VirtDeviceLUN;
- __le16 Unused;
- __le32 ReservedForFutureGrowth[68];
+ __le32 feature_bits3;
+ __le32 supported_performance_modes;
+ u8 host_bus_type; /* uses HOST_BUS_TYPE_xxx defines */
+ u8 host_bus_width; /* actual width in bits or links */
+ u16 host_bus_speed; /* actual bus speed/link rate in MHz */
+ u8 max_rrc_drives; /* max. number of ITP-RRC drives/pool */
+ u8 max_disk_xtasks; /* max. possible num of DiskX Tasks */
+
+ u8 cpld_ver_loaded;
+ u8 cpld_ver_in_flash;
+
+ __le64 max_rrc_capacity;
+ __le32 compiled_max_hist_log_level;
+ u8 custom_board_name[12];
+ u16 supported_cntlr_mode; /* identify supported controller mode */
+ u16 reserved_for_future16;
+ __le32 supported_options3; /* reserved for future options */
+
+ __le16 virt_device_bus; /* virt. SCSI device for Thor */
+ __le16 virt_device_target;
+ __le16 virt_device_lun;
+ __le16 unused;
+ __le32 reserved_for_future_growth[68];
};
#define AAC_FEATURE_FALCON cpu_to_le32(0x00000010)
#define AAC_OPTION_VARIABLE_BLOCK_SIZE cpu_to_le32(0x00040000)
/* 240 simple volume support */
#define AAC_OPTION_SUPPORTED_240_VOLUMES cpu_to_le32(0x10000000)
+/*
+ * Supports FIB dump sync command send prior to IOP_RESET
+ */
+#define AAC_OPTION_SUPPORTED3_IOP_RESET_FIB_DUMP cpu_to_le32(0x00004000)
#define AAC_SIS_VERSION_V3 3
#define AAC_SIS_SLOT_UNKNOWN 0xFF
#define GET_DRIVER_BUFFER_PROPERTIES 0x00000023
#define RCV_TEMP_READINGS 0x00000025
#define GET_COMM_PREFERRED_SETTINGS 0x00000026
+#define IOP_RESET_FW_FIB_DUMP 0x00000034
#define IOP_RESET 0x00001000
#define IOP_RESET_ALWAYS 0x00001001
#define RE_INIT_ADAPTER 0x000000ee
#define fib_data(fibctx) ((void *)(fibctx)->hw_fib_va->data)
struct aac_dev *aac_init_adapter(struct aac_dev *dev);
void aac_src_access_devreg(struct aac_dev *dev, int mode);
+void aac_set_intx_mode(struct aac_dev *dev);
int aac_get_config_status(struct aac_dev *dev, int commit_flag);
int aac_get_containers(struct aac_dev *dev);
int aac_scsi_cmd(struct scsi_cmnd *cmd);
extern int update_interval;
extern int check_interval;
extern int aac_check_reset;
+extern int aac_fib_dump;
#endif
goto cleanup;
}
- chn = aac_logical_to_phys(user_srbcmd->channel);
+ chn = user_srbcmd->channel;
if (chn < AAC_MAX_BUSES && user_srbcmd->id < AAC_MAX_TARGETS &&
dev->hba_map[chn][user_srbcmd->id].devtype ==
AAC_DEVTYPE_NATIVE_RAW) {
dev->pdev->device == PMC_DEVICE_S8 ||
dev->pdev->device == PMC_DEVICE_S9) &&
dev->msi_enabled)
- aac_src_access_devreg(dev, AAC_ENABLE_INTX);
+ aac_set_intx_mode(dev);
return status;
}
void aac_fib_map_free(struct aac_dev *dev)
{
- if (dev->hw_fib_va && dev->max_cmd_size) {
- pci_free_consistent(dev->pdev,
- (dev->max_cmd_size *
- (dev->scsi_host_ptr->can_queue + AAC_NUM_MGT_FIB)),
- dev->hw_fib_va, dev->hw_fib_pa);
- }
+ size_t alloc_size;
+ size_t fib_size;
+ int num_fibs;
+
+ if(!dev->hw_fib_va || !dev->max_cmd_size)
+ return;
+
+ num_fibs = dev->scsi_host_ptr->can_queue + AAC_NUM_MGT_FIB;
+ fib_size = dev->max_fib_size + sizeof(struct aac_fib_xporthdr);
+ alloc_size = fib_size * num_fibs + ALIGN32 - 1;
+
+ pci_free_consistent(dev->pdev, alloc_size, dev->hw_fib_va,
+ dev->hw_fib_pa);
+
dev->hw_fib_va = NULL;
dev->hw_fib_pa = 0;
}
if (i<0)
return -ENOMEM;
- /* 32 byte alignment for PMC */
- hw_fib_pa = (dev->hw_fib_pa + (ALIGN32 - 1)) & ~(ALIGN32 - 1);
- dev->hw_fib_va = (struct hw_fib *)((unsigned char *)dev->hw_fib_va +
- (hw_fib_pa - dev->hw_fib_pa));
- dev->hw_fib_pa = hw_fib_pa;
memset(dev->hw_fib_va, 0,
(dev->max_cmd_size + sizeof(struct aac_fib_xporthdr)) *
(dev->scsi_host_ptr->can_queue + AAC_NUM_MGT_FIB));
+ /* 32 byte alignment for PMC */
+ hw_fib_pa = (dev->hw_fib_pa + (ALIGN32 - 1)) & ~(ALIGN32 - 1);
+ hw_fib = (struct hw_fib *)((unsigned char *)dev->hw_fib_va +
+ (hw_fib_pa - dev->hw_fib_pa));
+
/* add Xport header */
- dev->hw_fib_va = (struct hw_fib *)((unsigned char *)dev->hw_fib_va +
+ hw_fib = (struct hw_fib *)((unsigned char *)hw_fib +
sizeof(struct aac_fib_xporthdr));
- dev->hw_fib_pa += sizeof(struct aac_fib_xporthdr);
+ hw_fib_pa += sizeof(struct aac_fib_xporthdr);
- hw_fib = dev->hw_fib_va;
- hw_fib_pa = dev->hw_fib_pa;
/*
* Initialise the fibs
*/
return 0;
}
+#ifdef CONFIG_EEH
+static inline int aac_check_eeh_failure(struct aac_dev *dev)
+{
+ /* Check for an EEH failure for the given
+ * device node. Function eeh_dev_check_failure()
+ * returns 0 if there has not been an EEH error
+ * otherwise returns a non-zero value.
+ *
+ * Need to be called before any PCI operation,
+ * i.e.,before aac_adapter_check_health()
+ */
+ struct eeh_dev *edev = pci_dev_to_eeh_dev(dev->pdev);
+
+ if (eeh_dev_check_failure(edev)) {
+ /* The EEH mechanisms will handle this
+ * error and reset the device if
+ * necessary.
+ */
+ return 1;
+ }
+ return 0;
+}
+#else
+static inline int aac_check_eeh_failure(struct aac_dev *dev)
+{
+ return 0;
+}
+#endif
+
/*
* Define the highest level of host to adapter communication routines.
* These routines will support host to adapter FS commuication. These
unsigned long mflags = 0;
unsigned long sflags = 0;
-
if (!(hw_fib->header.XferState & cpu_to_le32(HostOwned)))
return -EBUSY;
+
+ if (hw_fib->header.XferState & cpu_to_le32(AdapterProcessed))
+ return -EINVAL;
+
/*
* There are 5 cases with the wait and response requested flags.
* The only invalid cases are if the caller requests to wait and
}
return -ETIMEDOUT;
}
+
+ if (aac_check_eeh_failure(dev))
+ return -EFAULT;
+
if ((blink = aac_adapter_check_health(dev)) > 0) {
if (wait == -1) {
printk(KERN_ERR "aacraid: aac_fib_send: adapter blinkLED 0x%x.\n"
FIB_COUNTER_INCREMENT(aac_config.NativeSent);
if (wait) {
+
spin_unlock_irqrestore(&fibptr->event_lock, flags);
+
+ if (aac_check_eeh_failure(dev))
+ return -EFAULT;
+
/* Only set for first known interruptable command */
if (down_interruptible(&fibptr->event_wait)) {
fibptr->done = 2;
command->SCp.phase = AAC_OWNER_ERROR_HANDLER;
command->scsi_done(command);
}
+ /*
+ * Any Device that was already marked offline needs to be cleaned up
+ */
+ __shost_for_each_device(dev, host) {
+ if (!scsi_device_online(dev)) {
+ sdev_printk(KERN_INFO, dev, "Removing offline device\n");
+ scsi_remove_device(dev);
+ scsi_device_put(dev);
+ }
+ }
retval = 0;
out:
aac->in_reset = 0;
scsi_unblock_requests(host);
+ /*
+ * Issue bus rescan to catch any configuration that might have
+ * occurred
+ */
+ if (!retval) {
+ dev_info(&aac->pdev->dev, "Issuing bus rescan\n");
+ scsi_scan_host(host);
+ }
if (jafo) {
spin_lock_irq(host->host_lock);
}
printk(KERN_ERR "%s: Host adapter BLINK LED 0x%x\n", aac->name, BlinkLED);
if (!aac_check_reset || ((aac_check_reset == 1) &&
- (aac->supplement_adapter_info.SupportedOptions2 &
+ (aac->supplement_adapter_info.supported_options2 &
AAC_OPTION_IGNORE_RESET)))
goto out;
host = aac->scsi_host_ptr;
for (bus = 0; bus < AAC_MAX_BUSES; bus++) {
for (target = 0; target < AAC_MAX_TARGETS; target++) {
- if (aac_phys_to_logical(bus) == ENCLOSURE_CHANNEL)
- continue;
-
if (bus == CONTAINER_CHANNEL)
channel = CONTAINER_CHANNEL;
else
sdev = scsi_device_lookup(dev->scsi_host_ptr, channel,
target, 0);
- if (!sdev && devtype)
+ if (!sdev && new_devtype)
scsi_add_device(dev->scsi_host_ptr, channel,
target, 0);
else if (sdev && new_devtype != devtype)
/* Thor AIF */
aac_handle_sa_aif(dev, fib);
aac_fib_adapter_complete(fib, (u16)sizeof(u32));
- continue;
+ goto free_fib;
}
/*
* We will process the FIB here or pass it to a
aac_fib_init(fibptr);
- vbus = (u32)le16_to_cpu(dev->supplement_adapter_info.VirtDeviceBus);
- vid = (u32)le16_to_cpu(dev->supplement_adapter_info.VirtDeviceTarget);
+ vbus = (u32)le16_to_cpu(dev->supplement_adapter_info.virt_device_bus);
+ vid = (u32)le16_to_cpu(dev->supplement_adapter_info.virt_device_target);
srbcmd = (struct aac_srb *)fib_data(fibptr);
/* Don't even try to talk to adapter if its sick */
ret = aac_check_health(dev);
- if (!dev->queues)
+ if (ret || !dev->queues)
break;
next_check_jiffies = jiffies
+ ((long)(unsigned)check_interval)
&& (now.tv_usec > (1000000 / HZ)))
difference = (((1000000 - now.tv_usec) * HZ)
+ 500000) / 1000000;
- else if (ret == 0) {
-
+ else {
if (now.tv_usec > 500000)
++now.tv_sec;
ret = aac_send_hosttime(dev, &now);
difference = (long)(unsigned)update_interval*HZ;
- } else {
- /* retry shortly */
- difference = 10 * HZ;
}
next_jiffies = jiffies + difference;
if (time_before(next_check_jiffies,next_jiffies))
* Adapters that support a register, instead of a commanded,
* reset.
*/
- if (((aac->supplement_adapter_info.SupportedOptions2 &
+ if (((aac->supplement_adapter_info.supported_options2 &
AAC_OPTION_MU_RESET) ||
- (aac->supplement_adapter_info.SupportedOptions2 &
+ (aac->supplement_adapter_info.supported_options2 &
AAC_OPTION_DOORBELL_RESET)) &&
aac_check_reset &&
((aac_check_reset != 1) ||
- !(aac->supplement_adapter_info.SupportedOptions2 &
+ !(aac->supplement_adapter_info.supported_options2 &
AAC_OPTION_IGNORE_RESET))) {
/* Bypass wait for command quiesce */
aac_reset_adapter(aac, 2, IOP_HWSOFT_RESET);
struct aac_dev *dev = (struct aac_dev*)class_to_shost(device)->hostdata;
int len;
- if (dev->supplement_adapter_info.AdapterTypeText[0]) {
- char * cp = dev->supplement_adapter_info.AdapterTypeText;
+ if (dev->supplement_adapter_info.adapter_type_text[0]) {
+ char *cp = dev->supplement_adapter_info.adapter_type_text;
while (*cp && *cp != ' ')
++cp;
while (*cp == ' ')
struct device_attribute *attr, char *buf)
{
struct aac_dev *dev = (struct aac_dev*)class_to_shost(device)->hostdata;
+ struct aac_supplement_adapter_info *sup_adap_info;
int len;
- if (dev->supplement_adapter_info.AdapterTypeText[0]) {
- char * cp = dev->supplement_adapter_info.AdapterTypeText;
+ sup_adap_info = &dev->supplement_adapter_info;
+ if (sup_adap_info->adapter_type_text[0]) {
+ char *cp = sup_adap_info->adapter_type_text;
while (*cp && *cp != ' ')
++cp;
len = snprintf(buf, PAGE_SIZE, "%.*s\n",
- (int)(cp - (char *)dev->supplement_adapter_info.AdapterTypeText),
- dev->supplement_adapter_info.AdapterTypeText);
+ (int)(cp - (char *)sup_adap_info->adapter_type_text),
+ sup_adap_info->adapter_type_text);
} else
len = snprintf(buf, PAGE_SIZE, "%s\n",
- aac_drivers[dev->cardtype].vname);
+ aac_drivers[dev->cardtype].vname);
return len;
}
"SAI_READ_CAPACITY_16\n");
if (dev->jbod)
len += snprintf(buf + len, PAGE_SIZE - len, "SUPPORTED_JBOD\n");
- if (dev->supplement_adapter_info.SupportedOptions2 &
+ if (dev->supplement_adapter_info.supported_options2 &
AAC_OPTION_POWER_MANAGEMENT)
len += snprintf(buf + len, PAGE_SIZE - len,
"SUPPORTED_POWER_MANAGEMENT\n");
return len;
}
+static ssize_t aac_show_driver_version(struct device *device,
+ struct device_attribute *attr,
+ char *buf)
+{
+ return snprintf(buf, PAGE_SIZE, "%s\n", aac_driver_version);
+}
+
static ssize_t aac_show_serial_number(struct device *device,
struct device_attribute *attr, char *buf)
{
len = snprintf(buf, 16, "%06X\n",
le32_to_cpu(dev->adapter_info.serial[0]));
if (len &&
- !memcmp(&dev->supplement_adapter_info.MfgPcbaSerialNo[
- sizeof(dev->supplement_adapter_info.MfgPcbaSerialNo)-len],
+ !memcmp(&dev->supplement_adapter_info.mfg_pcba_serial_no[
+ sizeof(dev->supplement_adapter_info.mfg_pcba_serial_no)-len],
buf, len-1))
len = snprintf(buf, 16, "%.*s\n",
- (int)sizeof(dev->supplement_adapter_info.MfgPcbaSerialNo),
- dev->supplement_adapter_info.MfgPcbaSerialNo);
+ (int)sizeof(dev->supplement_adapter_info.mfg_pcba_serial_no),
+ dev->supplement_adapter_info.mfg_pcba_serial_no);
return min(len, 16);
}
},
.show = aac_show_bios_version,
};
+static struct device_attribute aac_lld_version = {
+ .attr = {
+ .name = "driver_version",
+ .mode = 0444,
+ },
+ .show = aac_show_driver_version,
+};
static struct device_attribute aac_serial_number = {
.attr = {
.name = "serial_number",
&aac_kernel_version,
&aac_monitor_version,
&aac_bios_version,
+ &aac_lld_version,
&aac_serial_number,
&aac_max_channel,
&aac_max_id,
{
u32 var = 0;
- if (!(dev->supplement_adapter_info.SupportedOptions2 &
+ if (!(dev->supplement_adapter_info.supported_options2 &
AAC_OPTION_MU_RESET) || (bled >= 0) || (bled == -2)) {
if (bled)
printk(KERN_ERR "%s%d: adapter kernel panic'd %x.\n",
{
u32 status = src_readl(dev, MUnit.OMR);
+ /*
+ * Check to see if the board panic'd.
+ */
+ if (unlikely(status & KERNEL_PANIC))
+ goto err_blink;
+
/*
* Check to see if the board failed any self tests.
*/
if (unlikely(status & SELF_TEST_FAILED))
- return -1;
+ goto err_out;
/*
- * Check to see if the board panic'd.
+ * Check to see if the board failed any self tests.
*/
- if (unlikely(status & KERNEL_PANIC))
- return (status >> 16) & 0xFF;
+ if (unlikely(status & MONITOR_PANIC))
+ goto err_out;
+
/*
* Wait for the adapter to be up and running.
*/
* Everything is OK
*/
return 0;
+
+err_out:
+ return -1;
+
+err_blink:
+ return (status > 16) & 0xFF;
}
static inline u32 aac_get_vector(struct aac_dev *dev)
return 0;
}
-static void aac_set_intx_mode(struct aac_dev *dev)
+void aac_set_intx_mode(struct aac_dev *dev)
{
if (dev->msi_enabled) {
aac_src_access_devreg(dev, AAC_ENABLE_INTX);
}
}
+static void aac_dump_fw_fib_iop_reset(struct aac_dev *dev)
+{
+ __le32 supported_options3;
+
+ if (!aac_fib_dump)
+ return;
+
+ supported_options3 = dev->supplement_adapter_info.supported_options3;
+ if (!(supported_options3 & AAC_OPTION_SUPPORTED3_IOP_RESET_FIB_DUMP))
+ return;
+
+ aac_adapter_sync_cmd(dev, IOP_RESET_FW_FIB_DUMP,
+ 0, 0, 0, 0, 0, 0, NULL, NULL, NULL, NULL, NULL);
+}
+
static void aac_send_iop_reset(struct aac_dev *dev, int bled)
{
u32 var, reset_mask;
+ aac_dump_fw_fib_iop_reset(dev);
+
bled = aac_adapter_sync_cmd(dev, IOP_RESET_ALWAYS,
0, 0, 0, 0, 0, 0, &var,
&reset_mask, NULL, NULL, NULL);
aac_set_intx_mode(dev);
- if (!bled && (dev->supplement_adapter_info.SupportedOptions2 &
+ if (!bled && (dev->supplement_adapter_info.supported_options2 &
AAC_OPTION_DOORBELL_RESET)) {
src_writel(dev, MUnit.IDR, reset_mask);
} else {
pr_err("%s%d: adapter kernel panic'd %x.\n",
dev->name, dev->id, bled);
+ /*
+ * When there is a BlinkLED, IOP_RESET has not effect
+ */
+ if (bled >= 2 && dev->sa_firmware && reset_type & HW_IOP_RESET)
+ reset_type &= ~HW_IOP_RESET;
+
dev->a_ops.adapter_enable_int = aac_src_disable_interrupt;
switch (reset_type) {
* does not disable its parity logic prior to
* the start of the reset. This may cause a
* parity error to be detected and thus a
- * spurious SERR or PERR assertion. Disble
+ * spurious SERR or PERR assertion. Disable
* PERR and SERR responses during the CHIPRST.
*/
mod_cmd = cmd & ~(PCIM_CMD_PERRESPEN|PCIM_CMD_SERRESPEN);
#include <linux/bitops.h>
#include <linux/log2.h>
#include <linux/interrupt.h>
-#include <linux/sched.h>
+#include <linux/sched/signal.h>
#include <linux/io.h>
#include <scsi/scsi.h>
#include <linux/spinlock.h>
#include <linux/interrupt.h>
#include <linux/delay.h>
-#include <linux/sched.h>
+#include <linux/sched/signal.h>
#include <linux/in.h>
#include <linux/kfifo.h>
#include <linux/netdevice.h>
0ULL };
static struct dev_dependent_vals dev_flash_gt_vals = { CXLFLASH_MAX_SECTORS,
CXLFLASH_NOTIFY_SHUTDOWN };
+static struct dev_dependent_vals dev_briard_vals = { CXLFLASH_MAX_SECTORS,
+ CXLFLASH_NOTIFY_SHUTDOWN };
/*
* PCI device binding table
PCI_ANY_ID, PCI_ANY_ID, 0, 0, (kernel_ulong_t)&dev_corsa_vals},
{PCI_VENDOR_ID_IBM, PCI_DEVICE_ID_IBM_FLASH_GT,
PCI_ANY_ID, PCI_ANY_ID, 0, 0, (kernel_ulong_t)&dev_flash_gt_vals},
+ {PCI_VENDOR_ID_IBM, PCI_DEVICE_ID_IBM_BRIARD,
+ PCI_ANY_ID, PCI_ANY_ID, 0, 0, (kernel_ulong_t)&dev_briard_vals},
{}
};
#define PCI_DEVICE_ID_IBM_CORSA 0x04F0
#define PCI_DEVICE_ID_IBM_FLASH_GT 0x0600
+#define PCI_DEVICE_ID_IBM_BRIARD 0x0624
/* Since there is only one target, make it 0 */
#define CXLFLASH_TARGET 0
struct cxlflash_cfg *cfg = shost_priv(sdev->host);
struct device *dev = &cfg->dev->dev;
struct glun_info *gli = lli->parent;
+ struct scsi_sense_hdr sshdr;
u8 *cmd_buf = NULL;
u8 *scsi_cmd = NULL;
u8 *sense_buf = NULL;
/* Drop the ioctl read semahpore across lengthy call */
up_read(&cfg->ioctl_rwsem);
result = scsi_execute(sdev, scsi_cmd, DMA_FROM_DEVICE, cmd_buf,
- CMD_BUFSIZE, sense_buf, to, CMD_RETRIES, 0, NULL);
+ CMD_BUFSIZE, sense_buf, &sshdr, to, CMD_RETRIES,
+ 0, 0, NULL);
down_read(&cfg->ioctl_rwsem);
rc = check_state(cfg);
if (rc) {
if (driver_byte(result) == DRIVER_SENSE) {
result &= ~(0xFF<<24); /* DRIVER_SENSE is not an error */
if (result & SAM_STAT_CHECK_CONDITION) {
- struct scsi_sense_hdr sshdr;
-
- scsi_normalize_sense(sense_buf, SCSI_SENSE_BUFFERSIZE,
- &sshdr);
switch (sshdr.sense_key) {
case NO_SENSE:
case RECOVERED_ERROR:
/* Drop the ioctl read semahpore across lengthy call */
up_read(&cfg->ioctl_rwsem);
result = scsi_execute(sdev, scsi_cmd, DMA_TO_DEVICE, cmd_buf,
- CMD_BUFSIZE, sense_buf, to, CMD_RETRIES,
- 0, NULL);
+ CMD_BUFSIZE, sense_buf, NULL, to,
+ CMD_RETRIES, 0, 0, NULL);
down_read(&cfg->ioctl_rwsem);
rc = check_state(cfg);
if (rc) {
cdb[1] = MI_REPORT_TARGET_PGS;
put_unaligned_be32(bufflen, &cdb[6]);
- return scsi_execute_req_flags(sdev, cdb, DMA_FROM_DEVICE,
- buff, bufflen, sshdr,
- ALUA_FAILOVER_TIMEOUT * HZ,
- ALUA_FAILOVER_RETRIES, NULL,
- req_flags, 0);
+ return scsi_execute(sdev, cdb, DMA_FROM_DEVICE, buff, bufflen, NULL,
+ sshdr, ALUA_FAILOVER_TIMEOUT * HZ,
+ ALUA_FAILOVER_RETRIES, req_flags, 0, NULL);
}
/*
cdb[1] = MO_SET_TARGET_PGS;
put_unaligned_be32(stpg_len, &cdb[6]);
- return scsi_execute_req_flags(sdev, cdb, DMA_TO_DEVICE,
- stpg_data, stpg_len,
- sshdr, ALUA_FAILOVER_TIMEOUT * HZ,
- ALUA_FAILOVER_RETRIES, NULL,
- req_flags, 0);
+ return scsi_execute(sdev, cdb, DMA_TO_DEVICE, stpg_data, stpg_len, NULL,
+ sshdr, ALUA_FAILOVER_TIMEOUT * HZ,
+ ALUA_FAILOVER_RETRIES, req_flags, 0, NULL);
}
static struct alua_port_group *alua_find_get_pg(char *id_str, size_t id_size,
BUG_ON((len > CLARIION_BUFFER_SIZE));
memcpy(csdev->buffer, page22, len);
- err = scsi_execute_req_flags(sdev, cdb, DMA_TO_DEVICE,
- csdev->buffer, len, &sshdr,
- CLARIION_TIMEOUT * HZ, CLARIION_RETRIES,
- NULL, req_flags, 0);
+ err = scsi_execute(sdev, cdb, DMA_TO_DEVICE, csdev->buffer, len, NULL,
+ &sshdr, CLARIION_TIMEOUT * HZ, CLARIION_RETRIES,
+ req_flags, 0, NULL);
if (err) {
if (scsi_sense_valid(&sshdr))
res = trespass_endio(sdev, &sshdr);
static int clariion_std_inquiry(struct scsi_device *sdev,
struct clariion_dh_data *csdev)
{
- int err;
+ int err = SCSI_DH_OK;
char *sp_model;
sp_model = parse_sp_model(sdev, sdev->inquiry);
REQ_FAILFAST_DRIVER;
retry:
- res = scsi_execute_req_flags(sdev, cmd, DMA_NONE, NULL, 0, &sshdr,
- HP_SW_TIMEOUT, HP_SW_RETRIES,
- NULL, req_flags, 0);
+ res = scsi_execute(sdev, cmd, DMA_NONE, NULL, 0, NULL, &sshdr,
+ HP_SW_TIMEOUT, HP_SW_RETRIES, req_flags, 0, NULL);
if (res) {
if (scsi_sense_valid(&sshdr))
ret = tur_done(sdev, h, &sshdr);
REQ_FAILFAST_DRIVER;
retry:
- res = scsi_execute_req_flags(sdev, cmd, DMA_NONE, NULL, 0, &sshdr,
- HP_SW_TIMEOUT, HP_SW_RETRIES,
- NULL, req_flags, 0);
+ res = scsi_execute(sdev, cmd, DMA_NONE, NULL, 0, NULL, &sshdr,
+ HP_SW_TIMEOUT, HP_SW_RETRIES, req_flags, 0, NULL);
if (res) {
if (!scsi_sense_valid(&sshdr)) {
sdev_printk(KERN_WARNING, sdev,
(char *) h->ctlr->array_name, h->ctlr->index,
(retry_cnt == RDAC_RETRY_COUNT) ? "queueing" : "retrying");
- if (scsi_execute_req_flags(sdev, cdb, DMA_TO_DEVICE,
- &h->ctlr->mode_select, data_size, &sshdr,
- RDAC_TIMEOUT * HZ,
- RDAC_RETRIES, NULL, req_flags, 0)) {
+ if (scsi_execute(sdev, cdb, DMA_TO_DEVICE, &h->ctlr->mode_select,
+ data_size, NULL, &sshdr, RDAC_TIMEOUT * HZ,
+ RDAC_RETRIES, req_flags, 0, NULL)) {
err = mode_select_handle_sense(sdev, &sshdr);
if (err == SCSI_DH_RETRY && retry_cnt--)
goto retry;
#include <linux/slab.h>
#include <linux/err.h>
#include <linux/export.h>
+#include <linux/rculist.h>
+
#include <asm/unaligned.h>
#include <scsi/fc/fc_gs.h>
#include <linux/timer.h>
#include <linux/workqueue.h>
#include <linux/export.h>
+#include <linux/rculist.h>
+
#include <asm/unaligned.h>
#include <scsi/libfc.h>
#include <linux/delay.h>
#include <linux/log2.h>
#include <linux/slab.h>
+#include <linux/sched/signal.h>
#include <linux/module.h>
#include <asm/unaligned.h>
#include <net/tcp.h>
#/*******************************************************************
# * This file is part of the Emulex Linux Device Driver for *
# * Fibre Channel Host Bus Adapters. *
+# * Copyright (C) 2017 Broadcom. All Rights Reserved. The term *
+# * “Broadcom” refers to Broadcom Limited and/or its subsidiaries. *
# * Copyright (C) 2004-2012 Emulex. All rights reserved. *
# * EMULEX and SLI are trademarks of Emulex. *
-# * www.emulex.com *
+# * www.broadcom.com *
# * *
# * This program is free software; you can redistribute it and/or *
# * modify it under the terms of version 2 of the GNU General *
obj-$(CONFIG_SCSI_LPFC) := lpfc.o
-lpfc-objs := lpfc_mem.o lpfc_sli.o lpfc_ct.o lpfc_els.o lpfc_hbadisc.o \
- lpfc_init.o lpfc_mbox.o lpfc_nportdisc.o lpfc_scsi.o lpfc_attr.o \
- lpfc_vport.o lpfc_debugfs.o lpfc_bsg.o
+lpfc-objs := lpfc_mem.o lpfc_sli.o lpfc_ct.o lpfc_els.o \
+ lpfc_hbadisc.o lpfc_init.o lpfc_mbox.o lpfc_nportdisc.o \
+ lpfc_scsi.o lpfc_attr.o lpfc_vport.o lpfc_debugfs.o lpfc_bsg.o \
+ lpfc_nvme.o lpfc_nvmet.o
/*******************************************************************
* This file is part of the Emulex Linux Device Driver for *
* Fibre Channel Host Bus Adapters. *
+ * Copyright (C) 2017 Broadcom. All Rights Reserved. The term *
+ * “Broadcom” refers to Broadcom Limited and/or its subsidiaries. *
* Copyright (C) 2004-2016 Emulex. All rights reserved. *
* EMULEX and SLI are trademarks of Emulex. *
- * www.emulex.com *
+ * www.broadcom.com *
* Portions Copyright (C) 2004-2005 Christoph Hellwig *
* *
* This program is free software; you can redistribute it and/or *
*******************************************************************/
#include <scsi/scsi_host.h>
+#include <linux/ktime.h>
#if defined(CONFIG_DEBUG_FS) && !defined(CONFIG_SCSI_LPFC_DEBUG_FS)
#define CONFIG_SCSI_LPFC_DEBUG_FS
#define LPFC_MAX_SG_SEG_CNT 4096 /* sg element count per scsi cmnd */
#define LPFC_MAX_SGL_SEG_CNT 512 /* SGL element count per scsi cmnd */
#define LPFC_MAX_BPL_SEG_CNT 4096 /* BPL element count per scsi cmnd */
+#define LPFC_MIN_NVME_SEG_CNT 254
#define LPFC_MAX_SGE_SIZE 0x80000000 /* Maximum data allowed in a SGE */
#define LPFC_IOCB_LIST_CNT 2250 /* list of IOCBs for fast-path usage. */
DISABLE_FCP_RING_INT = 0x2
};
+struct perf_prof {
+ uint16_t cmd_cpu[40];
+ uint16_t rsp_cpu[40];
+ uint16_t qh_cpu[40];
+ uint16_t wqidx[40];
+};
+
+/*
+ * Provide for FC4 TYPE x28 - NVME. The
+ * bit mask for FCP and NVME is 0x8 identically
+ * because they are 32 bit positions distance.
+ */
+#define LPFC_FC4_TYPE_BITMASK 0x00000100
+
/* Provide DMA memory definitions the driver uses per port instance. */
struct lpfc_dmabuf {
struct list_head list;
struct hbq_dmabuf {
struct lpfc_dmabuf hbuf;
struct lpfc_dmabuf dbuf;
- uint32_t size;
+ uint16_t total_size;
+ uint16_t bytes_recv;
uint32_t tag;
struct lpfc_cq_event cq_event;
unsigned long time_stamp;
+ void *context;
+};
+
+struct rqb_dmabuf {
+ struct lpfc_dmabuf hbuf;
+ struct lpfc_dmabuf dbuf;
+ uint16_t total_size;
+ uint16_t bytes_recv;
+ void *context;
+ struct lpfc_iocbq *iocbq;
+ struct lpfc_sglq *sglq;
+ struct lpfc_queue *hrq; /* ptr to associated Header RQ */
+ struct lpfc_queue *drq; /* ptr to associated Data RQ */
};
/* Priority bit. Set value to exceed low water mark in lpfc_mem. */
int32_t stopped; /* HBA has not been restarted since last ERATT */
uint8_t fc_linkspeed; /* Link speed after last READ_LA */
- uint32_t num_disc_nodes; /*in addition to hba_state */
+ uint32_t num_disc_nodes; /* in addition to hba_state */
+ uint32_t gidft_inp; /* cnt of outstanding GID_FTs */
uint32_t fc_nlp_cnt; /* outstanding NODELIST requests */
uint32_t fc_rscn_id_cnt; /* count of RSCNs payloads in list */
uint32_t cfg_max_scsicmpl_time;
uint32_t cfg_tgt_queue_depth;
uint32_t cfg_first_burst_size;
-
uint32_t dev_loss_tmo_changed;
struct fc_vport *fc_vport;
#ifdef CONFIG_SCSI_LPFC_DEBUG_FS
struct dentry *debug_disc_trc;
struct dentry *debug_nodelist;
+ struct dentry *debug_nvmestat;
+ struct dentry *debug_nvmektime;
+ struct dentry *debug_cpucheck;
struct dentry *vport_debugfs_root;
struct lpfc_debugfs_trc *disc_trc;
atomic_t disc_trc_cnt;
uint16_t fdmi_num_disc;
uint32_t fdmi_hba_mask;
uint32_t fdmi_port_mask;
+
+ /* There is a single nvme instance per vport. */
+ struct nvme_fc_local_port *localport;
+ uint8_t nvmei_support; /* driver supports NVME Initiator */
+ uint32_t last_fcp_wqidx;
};
struct hbq_s {
struct hbq_dmabuf *);
};
-#define LPFC_MAX_HBQS 4
/* this matches the position in the lpfc_hbq_defs array */
#define LPFC_ELS_HBQ 0
-#define LPFC_EXTRA_HBQ 1
+#define LPFC_MAX_HBQS 1
enum hba_temp_state {
HBA_NORMAL_TEMP,
* Firmware supports Forced Link Speed
* capability
*/
+#define HBA_NVME_IOQ_FLUSH 0x80000 /* NVME IO queues flushed. */
+
uint32_t fcp_ring_in_use; /* When polling test if intr-hndlr active*/
struct lpfc_dmabuf slim2p;
uint8_t wwpn[8];
uint32_t RandomData[7];
uint8_t fcp_embed_io;
+ uint8_t nvme_support; /* Firmware supports NVME */
+ uint8_t nvmet_support; /* driver supports NVMET */
+#define LPFC_NVMET_MAX_PORTS 32
uint8_t mds_diags_support;
/* HBA Config Parameters */
uint32_t cfg_fcp_imax;
uint32_t cfg_fcp_cpu_map;
uint32_t cfg_fcp_io_channel;
+ uint32_t cfg_suppress_rsp;
+ uint32_t cfg_nvme_oas;
+ uint32_t cfg_nvme_io_channel;
+ uint32_t cfg_nvmet_mrq;
+ uint32_t cfg_nvmet_mrq_post;
+ uint32_t cfg_enable_nvmet;
+ uint32_t cfg_nvme_enable_fb;
+ uint32_t cfg_nvmet_fb_size;
uint32_t cfg_total_seg_cnt;
uint32_t cfg_sg_seg_cnt;
uint32_t cfg_sg_dma_buf_size;
#define LPFC_FDMI_SUPPORT 1 /* FDMI supported? */
uint32_t cfg_enable_SmartSAN;
uint32_t cfg_enable_mds_diags;
+ uint32_t cfg_enable_fc4_type;
+ uint32_t cfg_xri_split;
+#define LPFC_ENABLE_FCP 1
+#define LPFC_ENABLE_NVME 2
+#define LPFC_ENABLE_BOTH 3
+ uint32_t io_channel_irqs; /* number of irqs for io channels */
+ struct nvmet_fc_target_port *targetport;
lpfc_vpd_t vpd; /* vital product data */
struct pci_dev *pcidev;
unsigned long data_flags;
uint32_t hbq_in_use; /* HBQs in use flag */
- struct list_head rb_pend_list; /* Received buffers to be processed */
uint32_t hbq_count; /* Count of configured HBQs */
struct hbq_s hbqs[LPFC_MAX_HBQS]; /* local copy of hbq indicies */
- atomic_t fcp_qidx; /* next work queue to post work to */
+ atomic_t fcp_qidx; /* next FCP WQ (RR Policy) */
+ atomic_t nvme_qidx; /* next NVME WQ (RR Policy) */
phys_addr_t pci_bar0_map; /* Physical address for PCI BAR0 */
phys_addr_t pci_bar1_map; /* Physical address for PCI BAR1 */
/*
* stat counters
*/
- uint64_t fc4InputRequests;
- uint64_t fc4OutputRequests;
- uint64_t fc4ControlRequests;
+ uint64_t fc4ScsiInputRequests;
+ uint64_t fc4ScsiOutputRequests;
+ uint64_t fc4ScsiControlRequests;
+ uint64_t fc4ScsiIoCmpls;
+ uint64_t fc4NvmeInputRequests;
+ uint64_t fc4NvmeOutputRequests;
+ uint64_t fc4NvmeControlRequests;
+ uint64_t fc4NvmeIoCmpls;
+ uint64_t fc4NvmeLsRequests;
+ uint64_t fc4NvmeLsCmpls;
+
uint64_t bg_guard_err_cnt;
uint64_t bg_apptag_err_cnt;
uint64_t bg_reftag_err_cnt;
struct list_head lpfc_scsi_buf_list_get;
struct list_head lpfc_scsi_buf_list_put;
uint32_t total_scsi_bufs;
+ spinlock_t nvme_buf_list_get_lock; /* NVME buf alloc list lock */
+ spinlock_t nvme_buf_list_put_lock; /* NVME buf free list lock */
+ struct list_head lpfc_nvme_buf_list_get;
+ struct list_head lpfc_nvme_buf_list_put;
+ uint32_t total_nvme_bufs;
struct list_head lpfc_iocb_list;
uint32_t total_iocbq_bufs;
struct list_head active_rrq_list;
spinlock_t hbalock;
/* pci_mem_pools */
- struct pci_pool *lpfc_scsi_dma_buf_pool;
+ struct pci_pool *lpfc_sg_dma_buf_pool;
struct pci_pool *lpfc_mbuf_pool;
struct pci_pool *lpfc_hrb_pool; /* header receive buffer pool */
struct pci_pool *lpfc_drb_pool; /* data receive buffer pool */
struct pci_pool *lpfc_hbq_pool; /* SLI3 hbq buffer pool */
+ struct pci_pool *txrdy_payload_pool;
struct lpfc_dma_pool lpfc_mbuf_safety_pool;
mempool_t *mbox_mem_pool;
enum intr_type_t intr_type;
uint32_t intr_mode;
#define LPFC_INTR_ERROR 0xFFFFFFFF
- struct msix_entry msix_entries[LPFC_MSIX_VECTORS];
-
struct list_head port_list;
struct lpfc_vport *pport; /* physical lpfc_vport pointer */
uint16_t max_vpi; /* Maximum virtual nports */
struct dentry *debug_readApp; /* inject read app_tag errors */
struct dentry *debug_readRef; /* inject read ref_tag errors */
+ struct dentry *debug_nvmeio_trc;
+ struct lpfc_debugfs_nvmeio_trc *nvmeio_trc;
+ atomic_t nvmeio_trc_cnt;
+ uint32_t nvmeio_trc_size;
+ uint32_t nvmeio_trc_output_idx;
+
/* T10 DIF error injection */
uint32_t lpfc_injerr_wgrd_cnt;
uint32_t lpfc_injerr_wapp_cnt;
struct dentry *idiag_ctl_acc;
struct dentry *idiag_mbx_acc;
struct dentry *idiag_ext_acc;
+ uint8_t lpfc_idiag_last_eq;
#endif
+ uint16_t nvmeio_trc_on;
/* Used for deferred freeing of ELS data buffers */
struct list_head elsbuf;
#define LPFC_TRANSGRESSION_LOW_RXPOWER 0x4000
uint16_t sfp_alarm;
uint16_t sfp_warning;
+
+#ifdef CONFIG_SCSI_LPFC_DEBUG_FS
+#define LPFC_CHECK_CPU_CNT 32
+ uint32_t cpucheck_rcv_io[LPFC_CHECK_CPU_CNT];
+ uint32_t cpucheck_xmt_io[LPFC_CHECK_CPU_CNT];
+ uint32_t cpucheck_cmpl_io[LPFC_CHECK_CPU_CNT];
+ uint32_t cpucheck_ccmpl_io[LPFC_CHECK_CPU_CNT];
+ uint16_t cpucheck_on;
+#define LPFC_CHECK_OFF 0
+#define LPFC_CHECK_NVME_IO 1
+#define LPFC_CHECK_NVMET_RCV 2
+#define LPFC_CHECK_NVMET_IO 4
+ uint16_t ktime_on;
+ uint64_t ktime_data_samples;
+ uint64_t ktime_status_samples;
+ uint64_t ktime_last_cmd;
+ uint64_t ktime_seg1_total;
+ uint64_t ktime_seg1_min;
+ uint64_t ktime_seg1_max;
+ uint64_t ktime_seg2_total;
+ uint64_t ktime_seg2_min;
+ uint64_t ktime_seg2_max;
+ uint64_t ktime_seg3_total;
+ uint64_t ktime_seg3_min;
+ uint64_t ktime_seg3_max;
+ uint64_t ktime_seg4_total;
+ uint64_t ktime_seg4_min;
+ uint64_t ktime_seg4_max;
+ uint64_t ktime_seg5_total;
+ uint64_t ktime_seg5_min;
+ uint64_t ktime_seg5_max;
+ uint64_t ktime_seg6_total;
+ uint64_t ktime_seg6_min;
+ uint64_t ktime_seg6_max;
+ uint64_t ktime_seg7_total;
+ uint64_t ktime_seg7_min;
+ uint64_t ktime_seg7_max;
+ uint64_t ktime_seg8_total;
+ uint64_t ktime_seg8_min;
+ uint64_t ktime_seg8_max;
+ uint64_t ktime_seg9_total;
+ uint64_t ktime_seg9_min;
+ uint64_t ktime_seg9_max;
+ uint64_t ktime_seg10_total;
+ uint64_t ktime_seg10_min;
+ uint64_t ktime_seg10_max;
+#endif
};
static inline struct Scsi_Host *
return 0;
}
+
+static inline struct lpfc_sli_ring *
+lpfc_phba_elsring(struct lpfc_hba *phba)
+{
+ if (phba->sli_rev == LPFC_SLI_REV4)
+ return phba->sli4_hba.els_wq->pring;
+ return &phba->sli.sli3_ring[LPFC_ELS_RING];
+}
/*******************************************************************
* This file is part of the Emulex Linux Device Driver for *
* Fibre Channel Host Bus Adapters. *
+ * Copyright (C) 2017 Broadcom. All Rights Reserved. The term *
+ * “Broadcom” refers to Broadcom Limited and/or its subsidiaries. *
* Copyright (C) 2004-2016 Emulex. All rights reserved. *
* EMULEX and SLI are trademarks of Emulex. *
- * www.emulex.com *
+ * www.broadcom.com *
* Portions Copyright (C) 2004-2005 Christoph Hellwig *
* *
* This program is free software; you can redistribute it and/or *
#include <scsi/scsi_transport_fc.h>
#include <scsi/fc/fc_fs.h>
+#include <linux/nvme-fc-driver.h>
+
#include "lpfc_hw4.h"
#include "lpfc_hw.h"
#include "lpfc_sli.h"
#include "lpfc_sli4.h"
#include "lpfc_nl.h"
#include "lpfc_disc.h"
-#include "lpfc_scsi.h"
#include "lpfc.h"
+#include "lpfc_scsi.h"
+#include "lpfc_nvme.h"
+#include "lpfc_nvmet.h"
#include "lpfc_logmsg.h"
#include "lpfc_version.h"
#include "lpfc_compat.h"
#include "lpfc_vport.h"
#include "lpfc_attr.h"
-#define LPFC_DEF_DEVLOSS_TMO 30
-#define LPFC_MIN_DEVLOSS_TMO 1
-#define LPFC_MAX_DEVLOSS_TMO 255
+#define LPFC_DEF_DEVLOSS_TMO 30
+#define LPFC_MIN_DEVLOSS_TMO 1
+#define LPFC_MAX_DEVLOSS_TMO 255
+
+#define LPFC_DEF_MRQ_POST 256
+#define LPFC_MIN_MRQ_POST 32
+#define LPFC_MAX_MRQ_POST 512
/*
* Write key size should be multiple of 4. If write key is changed
return snprintf(buf, PAGE_SIZE, "0\n");
}
+static ssize_t
+lpfc_nvme_info_show(struct device *dev, struct device_attribute *attr,
+ char *buf)
+{
+ struct Scsi_Host *shost = class_to_shost(dev);
+ struct lpfc_vport *vport = shost_priv(shost);
+ struct lpfc_hba *phba = vport->phba;
+ struct lpfc_nvmet_tgtport *tgtp;
+ struct nvme_fc_local_port *localport;
+ struct lpfc_nvme_lport *lport;
+ struct lpfc_nvme_rport *rport;
+ struct nvme_fc_remote_port *nrport;
+ char *statep;
+ int len = 0;
+
+ if (!(phba->cfg_enable_fc4_type & LPFC_ENABLE_NVME)) {
+ len += snprintf(buf, PAGE_SIZE, "NVME Disabled\n");
+ return len;
+ }
+ if (phba->nvmet_support) {
+ if (!phba->targetport) {
+ len = snprintf(buf, PAGE_SIZE,
+ "NVME Target: x%llx is not allocated\n",
+ wwn_to_u64(vport->fc_portname.u.wwn));
+ return len;
+ }
+ /* Port state is only one of two values for now. */
+ if (phba->targetport->port_id)
+ statep = "REGISTERED";
+ else
+ statep = "INIT";
+ len += snprintf(buf + len, PAGE_SIZE - len,
+ "NVME Target: Enabled State %s\n",
+ statep);
+ len += snprintf(buf + len, PAGE_SIZE - len,
+ "%s%d WWPN x%llx WWNN x%llx DID x%06x\n",
+ "NVME Target: lpfc",
+ phba->brd_no,
+ wwn_to_u64(vport->fc_portname.u.wwn),
+ wwn_to_u64(vport->fc_nodename.u.wwn),
+ phba->targetport->port_id);
+
+ len += snprintf(buf + len, PAGE_SIZE,
+ "\nNVME Target: Statistics\n");
+ tgtp = (struct lpfc_nvmet_tgtport *)phba->targetport->private;
+ len += snprintf(buf+len, PAGE_SIZE-len,
+ "LS: Rcv %08x Drop %08x Abort %08x\n",
+ atomic_read(&tgtp->rcv_ls_req_in),
+ atomic_read(&tgtp->rcv_ls_req_drop),
+ atomic_read(&tgtp->xmt_ls_abort));
+ if (atomic_read(&tgtp->rcv_ls_req_in) !=
+ atomic_read(&tgtp->rcv_ls_req_out)) {
+ len += snprintf(buf+len, PAGE_SIZE-len,
+ "Rcv LS: in %08x != out %08x\n",
+ atomic_read(&tgtp->rcv_ls_req_in),
+ atomic_read(&tgtp->rcv_ls_req_out));
+ }
+
+ len += snprintf(buf+len, PAGE_SIZE-len,
+ "LS: Xmt %08x Drop %08x Cmpl %08x Err %08x\n",
+ atomic_read(&tgtp->xmt_ls_rsp),
+ atomic_read(&tgtp->xmt_ls_drop),
+ atomic_read(&tgtp->xmt_ls_rsp_cmpl),
+ atomic_read(&tgtp->xmt_ls_rsp_error));
+
+ len += snprintf(buf+len, PAGE_SIZE-len,
+ "FCP: Rcv %08x Drop %08x\n",
+ atomic_read(&tgtp->rcv_fcp_cmd_in),
+ atomic_read(&tgtp->rcv_fcp_cmd_drop));
+
+ if (atomic_read(&tgtp->rcv_fcp_cmd_in) !=
+ atomic_read(&tgtp->rcv_fcp_cmd_out)) {
+ len += snprintf(buf+len, PAGE_SIZE-len,
+ "Rcv FCP: in %08x != out %08x\n",
+ atomic_read(&tgtp->rcv_fcp_cmd_in),
+ atomic_read(&tgtp->rcv_fcp_cmd_out));
+ }
+
+ len += snprintf(buf+len, PAGE_SIZE-len,
+ "FCP Rsp: RD %08x rsp %08x WR %08x rsp %08x\n",
+ atomic_read(&tgtp->xmt_fcp_read),
+ atomic_read(&tgtp->xmt_fcp_read_rsp),
+ atomic_read(&tgtp->xmt_fcp_write),
+ atomic_read(&tgtp->xmt_fcp_rsp));
+
+ len += snprintf(buf+len, PAGE_SIZE-len,
+ "FCP Rsp: abort %08x drop %08x\n",
+ atomic_read(&tgtp->xmt_fcp_abort),
+ atomic_read(&tgtp->xmt_fcp_drop));
+
+ len += snprintf(buf+len, PAGE_SIZE-len,
+ "FCP Rsp Cmpl: %08x err %08x drop %08x\n",
+ atomic_read(&tgtp->xmt_fcp_rsp_cmpl),
+ atomic_read(&tgtp->xmt_fcp_rsp_error),
+ atomic_read(&tgtp->xmt_fcp_rsp_drop));
+
+ len += snprintf(buf+len, PAGE_SIZE-len,
+ "ABORT: Xmt %08x Err %08x Cmpl %08x",
+ atomic_read(&tgtp->xmt_abort_rsp),
+ atomic_read(&tgtp->xmt_abort_rsp_error),
+ atomic_read(&tgtp->xmt_abort_cmpl));
+
+ len += snprintf(buf+len, PAGE_SIZE-len, "\n");
+ return len;
+ }
+
+ localport = vport->localport;
+ if (!localport) {
+ len = snprintf(buf, PAGE_SIZE,
+ "NVME Initiator x%llx is not allocated\n",
+ wwn_to_u64(vport->fc_portname.u.wwn));
+ return len;
+ }
+ len = snprintf(buf, PAGE_SIZE, "NVME Initiator Enabled\n");
+
+ spin_lock_irq(shost->host_lock);
+ lport = (struct lpfc_nvme_lport *)localport->private;
+
+ /* Port state is only one of two values for now. */
+ if (localport->port_id)
+ statep = "ONLINE";
+ else
+ statep = "UNKNOWN ";
+
+ len += snprintf(buf + len, PAGE_SIZE - len,
+ "%s%d WWPN x%llx WWNN x%llx DID x%06x %s\n",
+ "NVME LPORT lpfc",
+ phba->brd_no,
+ wwn_to_u64(vport->fc_portname.u.wwn),
+ wwn_to_u64(vport->fc_nodename.u.wwn),
+ localport->port_id, statep);
+
+ list_for_each_entry(rport, &lport->rport_list, list) {
+ /* local short-hand pointer. */
+ nrport = rport->remoteport;
+
+ /* Port state is only one of two values for now. */
+ switch (nrport->port_state) {
+ case FC_OBJSTATE_ONLINE:
+ statep = "ONLINE";
+ break;
+ case FC_OBJSTATE_UNKNOWN:
+ statep = "UNKNOWN ";
+ break;
+ default:
+ statep = "UNSUPPORTED";
+ break;
+ }
+
+ /* Tab in to show lport ownership. */
+ len += snprintf(buf + len, PAGE_SIZE - len,
+ "NVME RPORT ");
+ if (phba->brd_no >= 10)
+ len += snprintf(buf + len, PAGE_SIZE - len, " ");
+
+ len += snprintf(buf + len, PAGE_SIZE - len, "WWPN x%llx ",
+ nrport->port_name);
+ len += snprintf(buf + len, PAGE_SIZE - len, "WWNN x%llx ",
+ nrport->node_name);
+ len += snprintf(buf + len, PAGE_SIZE - len, "DID x%06x ",
+ nrport->port_id);
+
+ switch (nrport->port_role) {
+ case FC_PORT_ROLE_NVME_INITIATOR:
+ len += snprintf(buf + len, PAGE_SIZE - len,
+ "INITIATOR ");
+ break;
+ case FC_PORT_ROLE_NVME_TARGET:
+ len += snprintf(buf + len, PAGE_SIZE - len,
+ "TARGET ");
+ break;
+ case FC_PORT_ROLE_NVME_DISCOVERY:
+ len += snprintf(buf + len, PAGE_SIZE - len,
+ "DISCOVERY ");
+ break;
+ default:
+ len += snprintf(buf + len, PAGE_SIZE - len,
+ "UNKNOWN_ROLE x%x",
+ nrport->port_role);
+ break;
+ }
+ len += snprintf(buf + len, PAGE_SIZE - len, "%s ", statep);
+ /* Terminate the string. */
+ len += snprintf(buf + len, PAGE_SIZE - len, "\n");
+ }
+ spin_unlock_irq(shost->host_lock);
+
+ len += snprintf(buf + len, PAGE_SIZE, "\nNVME Statistics\n");
+ len += snprintf(buf+len, PAGE_SIZE-len,
+ "LS: Xmt %016llx Cmpl %016llx\n",
+ phba->fc4NvmeLsRequests,
+ phba->fc4NvmeLsCmpls);
+
+ len += snprintf(buf+len, PAGE_SIZE-len,
+ "FCP: Rd %016llx Wr %016llx IO %016llx\n",
+ phba->fc4NvmeInputRequests,
+ phba->fc4NvmeOutputRequests,
+ phba->fc4NvmeControlRequests);
+
+ len += snprintf(buf+len, PAGE_SIZE-len,
+ " Cmpl %016llx\n", phba->fc4NvmeIoCmpls);
+
+ return len;
+}
+
static ssize_t
lpfc_bg_info_show(struct device *dev, struct device_attribute *attr,
char *buf)
return 0;
}
+int
+lpfc_emptyq_wait(struct lpfc_hba *phba, struct list_head *q, spinlock_t *lock)
+{
+ int cnt = 0;
+
+ spin_lock_irq(lock);
+ while (!list_empty(q)) {
+ spin_unlock_irq(lock);
+ msleep(20);
+ if (cnt++ > 250) { /* 5 secs */
+ lpfc_printf_log(phba, KERN_WARNING, LOG_INIT,
+ "0466 %s %s\n",
+ "Outstanding IO when ",
+ "bringing Adapter offline\n");
+ return 0;
+ }
+ spin_lock_irq(lock);
+ }
+ spin_unlock_irq(lock);
+ return 1;
+}
+
/**
* lpfc_do_offline - Issues a mailbox command to bring the link down
* @phba: lpfc_hba pointer.
lpfc_do_offline(struct lpfc_hba *phba, uint32_t type)
{
struct completion online_compl;
+ struct lpfc_queue *qp = NULL;
struct lpfc_sli_ring *pring;
struct lpfc_sli *psli;
int status = 0;
- int cnt = 0;
int i;
int rc;
/* Wait a little for things to settle down, but not
* long enough for dev loss timeout to expire.
*/
- for (i = 0; i < psli->num_rings; i++) {
- pring = &psli->ring[i];
- while (!list_empty(&pring->txcmplq)) {
- msleep(10);
- if (cnt++ > 500) { /* 5 secs */
- lpfc_printf_log(phba,
- KERN_WARNING, LOG_INIT,
- "0466 Outstanding IO when "
- "bringing Adapter offline\n");
- break;
- }
+ if (phba->sli_rev != LPFC_SLI_REV4) {
+ for (i = 0; i < psli->num_rings; i++) {
+ pring = &psli->sli3_ring[i];
+ if (!lpfc_emptyq_wait(phba, &pring->txcmplq,
+ &phba->hbalock))
+ goto out;
+ }
+ } else {
+ list_for_each_entry(qp, &phba->sli4_hba.lpfc_wq_list, wq_list) {
+ pring = qp->pring;
+ if (!pring)
+ continue;
+ if (!lpfc_emptyq_wait(phba, &pring->txcmplq,
+ &pring->ring_lock))
+ goto out;
}
}
-
+out:
init_completion(&online_compl);
rc = lpfc_workq_post_event(phba, &status, &online_compl, type);
if (rc == 0)
}
+static DEVICE_ATTR(nvme_info, 0444, lpfc_nvme_info_show, NULL);
static DEVICE_ATTR(bg_info, S_IRUGO, lpfc_bg_info_show, NULL);
static DEVICE_ATTR(bg_guard_err, S_IRUGO, lpfc_bg_guard_err_show, NULL);
static DEVICE_ATTR(bg_apptag_err, S_IRUGO, lpfc_bg_apptag_err_show, NULL);
static DEVICE_ATTR(lpfc_xlane_lun, S_IRUGO | S_IWUSR,
lpfc_oas_lun_show, lpfc_oas_lun_store);
+int lpfc_enable_nvmet_cnt;
+unsigned long long lpfc_enable_nvmet[LPFC_NVMET_MAX_PORTS] = {
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0};
+module_param_array(lpfc_enable_nvmet, ullong, &lpfc_enable_nvmet_cnt, 0444);
+MODULE_PARM_DESC(lpfc_enable_nvmet, "Enable HBA port(s) WWPN as a NVME Target");
+
static int lpfc_poll = 0;
module_param(lpfc_poll, int, S_IRUGO);
MODULE_PARM_DESC(lpfc_poll, "FCP ring polling mode control:"
{
struct Scsi_Host *shost = class_to_shost(dev);
struct lpfc_hba *phba = ((struct lpfc_vport *) shost->hostdata)->phba;
+ struct lpfc_sli_ring *pring = lpfc_phba_elsring(phba);
- return snprintf(buf, PAGE_SIZE, "%d\n",
- phba->sli.ring[LPFC_ELS_RING].txq_max);
+ return snprintf(buf, PAGE_SIZE, "%d\n", pring->txq_max);
}
static DEVICE_ATTR(txq_hw, S_IRUGO,
{
struct Scsi_Host *shost = class_to_shost(dev);
struct lpfc_hba *phba = ((struct lpfc_vport *) shost->hostdata)->phba;
+ struct lpfc_sli_ring *pring = lpfc_phba_elsring(phba);
- return snprintf(buf, PAGE_SIZE, "%d\n",
- phba->sli.ring[LPFC_ELS_RING].txcmplq_max);
+ return snprintf(buf, PAGE_SIZE, "%d\n", pring->txcmplq_max);
}
static DEVICE_ATTR(txcmplq_hw, S_IRUGO,
static DEVICE_ATTR(lpfc_devloss_tmo, S_IRUGO | S_IWUSR,
lpfc_devloss_tmo_show, lpfc_devloss_tmo_store);
+/*
+ * lpfc_suppress_rsp: Enable suppress rsp feature is firmware supports it
+ * lpfc_suppress_rsp = 0 Disable
+ * lpfc_suppress_rsp = 1 Enable (default)
+ *
+ */
+LPFC_ATTR_R(suppress_rsp, 1, 0, 1,
+ "Enable suppress rsp feature is firmware supports it");
+
+/*
+ * lpfc_nvmet_mrq: Specify number of RQ pairs for processing NVMET cmds
+ * lpfc_nvmet_mrq = 1 use a single RQ pair
+ * lpfc_nvmet_mrq >= 2 use specified RQ pairs for MRQ
+ *
+ */
+LPFC_ATTR_R(nvmet_mrq,
+ 1, 1, 16,
+ "Specify number of RQ pairs for processing NVMET cmds");
+
+/*
+ * lpfc_nvmet_mrq_post: Specify number buffers to post on every MRQ
+ *
+ */
+LPFC_ATTR_R(nvmet_mrq_post, LPFC_DEF_MRQ_POST,
+ LPFC_MIN_MRQ_POST, LPFC_MAX_MRQ_POST,
+ "Specify number of buffers to post on every MRQ");
+
+/*
+ * lpfc_enable_fc4_type: Defines what FC4 types are supported.
+ * Supported Values: 1 - register just FCP
+ * 3 - register both FCP and NVME
+ * Supported values are [1,3]. Default value is 3
+ */
+LPFC_ATTR_R(enable_fc4_type, LPFC_ENABLE_BOTH,
+ LPFC_ENABLE_FCP, LPFC_ENABLE_BOTH,
+ "Define fc4 type to register with fabric.");
+
+/*
+ * lpfc_xri_split: Defines the division of XRI resources between SCSI and NVME
+ * This parameter is only used if:
+ * lpfc_enable_fc4_type is 3 - register both FCP and NVME and
+ * port is not configured for NVMET.
+ *
+ * ELS/CT always get 10% of XRIs, up to a maximum of 250
+ * The remaining XRIs get split up based on lpfc_xri_split per port:
+ *
+ * Supported Values are in percentages
+ * the xri_split value is the percentage the SCSI port will get. The remaining
+ * percentage will go to NVME.
+ */
+LPFC_ATTR_R(xri_split, 50, 10, 90,
+ "Division of XRI resources between SCSI and NVME");
+
/*
# lpfc_log_verbose: Only turn this flag on if you are willing to risk being
# deluged with LOTS of information.
/*
* Value range for the HBA is [5000,5000000]
* The value for each EQ depends on how many EQs are configured.
+ * Allow value == 0
*/
- if (val < LPFC_MIN_IMAX || val > LPFC_MAX_IMAX)
+ if (val && (val < LPFC_MIN_IMAX || val > LPFC_MAX_IMAX))
return -EINVAL;
phba->cfg_fcp_imax = (uint32_t)val;
- for (i = 0; i < phba->cfg_fcp_io_channel; i += LPFC_MAX_EQ_DELAY)
- lpfc_modify_fcp_eq_delay(phba, i);
+ for (i = 0; i < phba->io_channel_irqs; i++)
+ lpfc_modify_hba_eq_delay(phba, i);
return strlen(buf);
}
return 0;
}
- if (val >= LPFC_MIN_IMAX && val <= LPFC_MAX_IMAX) {
+ if ((val >= LPFC_MIN_IMAX && val <= LPFC_MAX_IMAX) ||
+ (val == 0)) {
phba->cfg_fcp_imax = val;
return 0;
}
LPFC_VPORT_ATTR_RW(first_burst_size, 0, 0, 65536,
"First burst size for Targets that support first burst");
+/*
+* lpfc_nvmet_fb_size: NVME Target mode supported first burst size.
+* When the driver is configured as an NVME target, this value is
+* communicated to the NVME initiator in the PRLI response. It is
+* used only when the lpfc_nvme_enable_fb and lpfc_nvmet_support
+* parameters are set and the target is sending the PRLI RSP.
+* Parameter supported on physical port only - no NPIV support.
+* Value range is [0,65536]. Default value is 0.
+*/
+LPFC_ATTR_RW(nvmet_fb_size, 0, 0, 65536,
+ "NVME Target mode first burst size in 512B increments.");
+
+/*
+ * lpfc_nvme_enable_fb: Enable NVME first burst on I and T functions.
+ * For the Initiator (I), enabling this parameter means that an NVMET
+ * PRLI response with FBA enabled and an FB_SIZE set to a nonzero value will be
+ * processed by the initiator for subsequent NVME FCP IO. For the target
+ * function (T), enabling this parameter qualifies the lpfc_nvmet_fb_size
+ * driver parameter as the target function's first burst size returned to the
+ * initiator in the target's NVME PRLI response. Parameter supported on physical
+ * port only - no NPIV support.
+ * Value range is [0,1]. Default value is 0 (disabled).
+ */
+LPFC_ATTR_RW(nvme_enable_fb, 0, 0, 1,
+ "Enable First Burst feature on I and T functions.");
+
/*
# lpfc_max_scsicmpl_time: Use scsi command completion time to control I/O queue
# depth. Default value is 0. When the value of this parameter is zero the
LPFC_ATTR_R(ack0, 0, 0, 1, "Enable ACK0 support");
/*
-# lpfc_fcp_io_sched: Determine scheduling algrithmn for issuing FCP cmds
-# range is [0,1]. Default value is 0.
-# For [0], FCP commands are issued to Work Queues ina round robin fashion.
-# For [1], FCP commands are issued to a Work Queue associated with the
-# current CPU.
-# It would be set to 1 by the driver if it's able to set up cpu affinity
-# for FCP I/Os through Work Queue associated with the current CPU. Otherwise,
-# roundrobin scheduling of FCP I/Os through WQs will be used.
-*/
-LPFC_ATTR_RW(fcp_io_sched, 0, 0, 1, "Determine scheduling algorithm for "
- "issuing commands [0] - Round Robin, [1] - Current CPU");
+ * lpfc_io_sched: Determine scheduling algrithmn for issuing FCP cmds
+ * range is [0,1]. Default value is 0.
+ * For [0], FCP commands are issued to Work Queues ina round robin fashion.
+ * For [1], FCP commands are issued to a Work Queue associated with the
+ * current CPU.
+ *
+ * LPFC_FCP_SCHED_ROUND_ROBIN == 0
+ * LPFC_FCP_SCHED_BY_CPU == 1
+ *
+ * The driver dynamically sets this to 1 (BY_CPU) if it's able to set up cpu
+ * affinity for FCP/NVME I/Os through Work Queues associated with the current
+ * CPU. Otherwise, the default 0 (Round Robin) scheduling of FCP/NVME I/Os
+ * through WQs will be used.
+ */
+LPFC_ATTR_RW(fcp_io_sched, LPFC_FCP_SCHED_ROUND_ROBIN,
+ LPFC_FCP_SCHED_ROUND_ROBIN,
+ LPFC_FCP_SCHED_BY_CPU,
+ "Determine scheduling algorithm for "
+ "issuing commands [0] - Round Robin, [1] - Current CPU");
/*
# lpfc_fcp2_no_tgt_reset: Determine bus reset behavior
"MSI-X (2), if possible");
/*
-# lpfc_fcp_io_channel: Set the number of FCP EQ/CQ/WQ IO channels
-#
-# Value range is [1,7]. Default value is 4.
-*/
-LPFC_ATTR_R(fcp_io_channel, LPFC_FCP_IO_CHAN_DEF, LPFC_FCP_IO_CHAN_MIN,
- LPFC_FCP_IO_CHAN_MAX,
+ * lpfc_nvme_oas: Use the oas bit when sending NVME/NVMET IOs
+ *
+ * 0 = NVME OAS disabled
+ * 1 = NVME OAS enabled
+ *
+ * Value range is [0,1]. Default value is 0.
+ */
+LPFC_ATTR_RW(nvme_oas, 0, 0, 1,
+ "Use OAS bit on NVME IOs");
+
+/*
+ * lpfc_fcp_io_channel: Set the number of FCP IO channels the driver
+ * will advertise it supports to the SCSI layer. This also will map to
+ * the number of WQs the driver will create.
+ *
+ * 0 = Configure the number of io channels to the number of active CPUs.
+ * 1,32 = Manually specify how many io channels to use.
+ *
+ * Value range is [0,32]. Default value is 4.
+ */
+LPFC_ATTR_R(fcp_io_channel,
+ LPFC_FCP_IO_CHAN_DEF,
+ LPFC_HBA_IO_CHAN_MIN, LPFC_HBA_IO_CHAN_MAX,
"Set the number of FCP I/O channels");
+/*
+ * lpfc_nvme_io_channel: Set the number of IO hardware queues the driver
+ * will advertise it supports to the NVME layer. This also will map to
+ * the number of WQs the driver will create.
+ *
+ * This module parameter is valid when lpfc_enable_fc4_type is set
+ * to support NVME.
+ *
+ * The NVME Layer will try to create this many, plus 1 administrative
+ * hardware queue. The administrative queue will always map to WQ 0
+ * A hardware IO queue maps (qidx) to a specific driver WQ.
+ *
+ * 0 = Configure the number of io channels to the number of active CPUs.
+ * 1,32 = Manually specify how many io channels to use.
+ *
+ * Value range is [0,32]. Default value is 0.
+ */
+LPFC_ATTR_R(nvme_io_channel,
+ LPFC_NVME_IO_CHAN_DEF,
+ LPFC_HBA_IO_CHAN_MIN, LPFC_HBA_IO_CHAN_MAX,
+ "Set the number of NVME I/O channels");
+
/*
# lpfc_enable_hba_reset: Allow or prevent HBA resets to the hardware.
# 0 = HBA resets disabled
LPFC_ATTR_R(enable_mds_diags, 0, 0, 1, "Enable MDS Diagnostics");
struct device_attribute *lpfc_hba_attrs[] = {
+ &dev_attr_nvme_info,
&dev_attr_bg_info,
&dev_attr_bg_guard_err,
&dev_attr_bg_apptag_err,
&dev_attr_lpfc_peer_port_login,
&dev_attr_lpfc_nodev_tmo,
&dev_attr_lpfc_devloss_tmo,
+ &dev_attr_lpfc_enable_fc4_type,
+ &dev_attr_lpfc_xri_split,
&dev_attr_lpfc_fcp_class,
&dev_attr_lpfc_use_adisc,
&dev_attr_lpfc_first_burst_size,
&dev_attr_lpfc_poll_tmo,
&dev_attr_lpfc_task_mgmt_tmo,
&dev_attr_lpfc_use_msi,
+ &dev_attr_lpfc_nvme_oas,
&dev_attr_lpfc_fcp_imax,
&dev_attr_lpfc_fcp_cpu_map,
&dev_attr_lpfc_fcp_io_channel,
+ &dev_attr_lpfc_suppress_rsp,
+ &dev_attr_lpfc_nvme_io_channel,
+ &dev_attr_lpfc_nvmet_mrq,
+ &dev_attr_lpfc_nvmet_mrq_post,
+ &dev_attr_lpfc_nvme_enable_fb,
+ &dev_attr_lpfc_nvmet_fb_size,
&dev_attr_lpfc_enable_bg,
&dev_attr_lpfc_soft_wwnn,
&dev_attr_lpfc_soft_wwpn,
lpfc_fdmi_on_init(phba, lpfc_fdmi_on);
lpfc_enable_SmartSAN_init(phba, lpfc_enable_SmartSAN);
lpfc_use_msi_init(phba, lpfc_use_msi);
+ lpfc_nvme_oas_init(phba, lpfc_nvme_oas);
lpfc_fcp_imax_init(phba, lpfc_fcp_imax);
lpfc_fcp_cpu_map_init(phba, lpfc_fcp_cpu_map);
- lpfc_fcp_io_channel_init(phba, lpfc_fcp_io_channel);
lpfc_enable_hba_reset_init(phba, lpfc_enable_hba_reset);
lpfc_enable_hba_heartbeat_init(phba, lpfc_enable_hba_heartbeat);
+
lpfc_EnableXLane_init(phba, lpfc_EnableXLane);
if (phba->sli_rev != LPFC_SLI_REV4)
phba->cfg_EnableXLane = 0;
lpfc_XLanePriority_init(phba, lpfc_XLanePriority);
+
memset(phba->cfg_oas_tgt_wwpn, 0, (8 * sizeof(uint8_t)));
memset(phba->cfg_oas_vpt_wwpn, 0, (8 * sizeof(uint8_t)));
phba->cfg_oas_lun_state = 0;
phba->cfg_poll = 0;
else
phba->cfg_poll = lpfc_poll;
+ lpfc_suppress_rsp_init(phba, lpfc_suppress_rsp);
+
+ lpfc_enable_fc4_type_init(phba, lpfc_enable_fc4_type);
+ lpfc_nvmet_mrq_init(phba, lpfc_nvmet_mrq);
+ lpfc_nvmet_mrq_post_init(phba, lpfc_nvmet_mrq_post);
+
+ /* Initialize first burst. Target vs Initiator are different. */
+ lpfc_nvme_enable_fb_init(phba, lpfc_nvme_enable_fb);
+ lpfc_nvmet_fb_size_init(phba, lpfc_nvmet_fb_size);
+ lpfc_fcp_io_channel_init(phba, lpfc_fcp_io_channel);
+ lpfc_nvme_io_channel_init(phba, lpfc_nvme_io_channel);
+
+ if (phba->sli_rev != LPFC_SLI_REV4) {
+ /* NVME only supported on SLI4 */
+ phba->nvmet_support = 0;
+ phba->cfg_enable_fc4_type = LPFC_ENABLE_FCP;
+ } else {
+ /* We MUST have FCP support */
+ if (!(phba->cfg_enable_fc4_type & LPFC_ENABLE_FCP))
+ phba->cfg_enable_fc4_type |= LPFC_ENABLE_FCP;
+ }
+
+ /* A value of 0 means use the number of CPUs found in the system */
+ if (phba->cfg_fcp_io_channel == 0)
+ phba->cfg_fcp_io_channel = phba->sli4_hba.num_present_cpu;
+ if (phba->cfg_nvme_io_channel == 0)
+ phba->cfg_nvme_io_channel = phba->sli4_hba.num_present_cpu;
+
+ if (phba->cfg_enable_fc4_type == LPFC_ENABLE_NVME)
+ phba->cfg_fcp_io_channel = 0;
+
+ if (phba->cfg_enable_fc4_type == LPFC_ENABLE_FCP)
+ phba->cfg_nvme_io_channel = 0;
+
+ if (phba->cfg_fcp_io_channel > phba->cfg_nvme_io_channel)
+ phba->io_channel_irqs = phba->cfg_fcp_io_channel;
+ else
+ phba->io_channel_irqs = phba->cfg_nvme_io_channel;
phba->cfg_soft_wwnn = 0L;
phba->cfg_soft_wwpn = 0L;
+ lpfc_xri_split_init(phba, lpfc_xri_split);
lpfc_sg_seg_cnt_init(phba, lpfc_sg_seg_cnt);
lpfc_hba_queue_depth_init(phba, lpfc_hba_queue_depth);
lpfc_hba_log_verbose_init(phba, lpfc_log_verbose);
return;
}
+/**
+ * lpfc_nvme_mod_param_dep - Adjust module parameter value based on
+ * dependencies between protocols and roles.
+ * @phba: lpfc_hba pointer.
+ **/
+void
+lpfc_nvme_mod_param_dep(struct lpfc_hba *phba)
+{
+ if (phba->cfg_nvme_io_channel > phba->sli4_hba.num_present_cpu)
+ phba->cfg_nvme_io_channel = phba->sli4_hba.num_present_cpu;
+
+ if (phba->cfg_fcp_io_channel > phba->sli4_hba.num_present_cpu)
+ phba->cfg_fcp_io_channel = phba->sli4_hba.num_present_cpu;
+
+ if (phba->cfg_enable_fc4_type & LPFC_ENABLE_NVME &&
+ phba->nvmet_support) {
+ phba->cfg_enable_fc4_type &= ~LPFC_ENABLE_FCP;
+ phba->cfg_fcp_io_channel = 0;
+
+ lpfc_printf_log(phba, KERN_INFO, LOG_NVME_DISC,
+ "6013 %s x%x fb_size x%x, fb_max x%x\n",
+ "NVME Target PRLI ACC enable_fb ",
+ phba->cfg_nvme_enable_fb,
+ phba->cfg_nvmet_fb_size,
+ LPFC_NVMET_FB_SZ_MAX);
+
+ if (phba->cfg_nvme_enable_fb == 0)
+ phba->cfg_nvmet_fb_size = 0;
+ else {
+ if (phba->cfg_nvmet_fb_size > LPFC_NVMET_FB_SZ_MAX)
+ phba->cfg_nvmet_fb_size = LPFC_NVMET_FB_SZ_MAX;
+ }
+
+ /* Adjust lpfc_nvmet_mrq to avoid running out of WQE slots */
+ if (phba->cfg_nvmet_mrq > phba->cfg_nvme_io_channel) {
+ phba->cfg_nvmet_mrq = phba->cfg_nvme_io_channel;
+ lpfc_printf_log(phba, KERN_ERR, LOG_NVME_DISC,
+ "6018 Adjust lpfc_nvmet_mrq to %d\n",
+ phba->cfg_nvmet_mrq);
+ }
+ } else {
+ /* Not NVME Target mode. Turn off Target parameters. */
+ phba->nvmet_support = 0;
+ phba->cfg_nvmet_mrq = 0;
+ phba->cfg_nvmet_mrq_post = 0;
+ phba->cfg_nvmet_fb_size = 0;
+ }
+
+ if (phba->cfg_fcp_io_channel > phba->cfg_nvme_io_channel)
+ phba->io_channel_irqs = phba->cfg_fcp_io_channel;
+ else
+ phba->io_channel_irqs = phba->cfg_nvme_io_channel;
+}
+
/**
* lpfc_get_vport_cfgparam - Used during port create, init the vport structure
* @vport: lpfc_vport pointer.
/*******************************************************************
* This file is part of the Emulex Linux Device Driver for *
* Fibre Channel Host Bus Adapters. *
+ * Copyright (C) 2017 Broadcom. All Rights Reserved. The term *
+ * “Broadcom” refers to Broadcom Limited and/or its subsidiaries. *
* Copyright (C) 2004-2016 Emulex. All rights reserved. *
* EMULEX and SLI are trademarks of Emulex. *
- * www.emulex.com *
+ * www.broadcom.com *
* Portions Copyright (C) 2004-2005 Christoph Hellwig *
* *
* This program is free software; you can redistribute it and/or *
/*******************************************************************
* This file is part of the Emulex Linux Device Driver for *
* Fibre Channel Host Bus Adapters. *
+ * Copyright (C) 2017 Broadcom. All Rights Reserved. The term *
+ * “Broadcom” refers to Broadcom Limited and/or its subsidiaries. *
* Copyright (C) 2009-2015 Emulex. All rights reserved. *
* EMULEX and SLI are trademarks of Emulex. *
- * www.emulex.com *
+ * www.broadcom.com *
* *
* This program is free software; you can redistribute it and/or *
* modify it under the terms of version 2 of the GNU General *
struct lpfc_vport **vports;
struct Scsi_Host *shost;
struct lpfc_sli *psli;
+ struct lpfc_queue *qp = NULL;
struct lpfc_sli_ring *pring;
int i = 0;
if (!psli)
return -ENODEV;
- pring = &psli->ring[LPFC_FCP_RING];
- if (!pring)
- return -ENODEV;
if ((phba->link_state == LPFC_HBA_ERROR) ||
(psli->sli_flag & LPFC_BLOCK_MGMT_IO) ||
scsi_block_requests(shost);
}
- while (!list_empty(&pring->txcmplq)) {
- if (i++ > 500) /* wait up to 5 seconds */
+ if (phba->sli_rev != LPFC_SLI_REV4) {
+ pring = &psli->sli3_ring[LPFC_FCP_RING];
+ lpfc_emptyq_wait(phba, &pring->txcmplq, &phba->hbalock);
+ return 0;
+ }
+ list_for_each_entry(qp, &phba->sli4_hba.lpfc_wq_list, wq_list) {
+ pring = qp->pring;
+ if (!pring || (pring->ringno != LPFC_FCP_RING))
+ continue;
+ if (!lpfc_emptyq_wait(phba, &pring->txcmplq,
+ &pring->ring_lock))
break;
- msleep(10);
}
return 0;
}
* lpfc_bsg_dma_page_alloc - allocate a bsg mbox page sized dma buffers
* @phba: Pointer to HBA context object
*
- * This function allocates BSG_MBOX_SIZE (4KB) page size dma buffer and.
+ * This function allocates BSG_MBOX_SIZE (4KB) page size dma buffer and
* returns the pointer to the buffer.
**/
static struct lpfc_dmabuf *
static int lpfcdiag_loop_post_rxbufs(struct lpfc_hba *phba, uint16_t rxxri,
size_t len)
{
- struct lpfc_sli *psli = &phba->sli;
- struct lpfc_sli_ring *pring = &psli->ring[LPFC_ELS_RING];
+ struct lpfc_sli_ring *pring;
struct lpfc_iocbq *cmdiocbq;
IOCB_t *cmd = NULL;
struct list_head head, *curr, *next;
int iocb_stat;
int i = 0;
+ pring = lpfc_phba_elsring(phba);
+
cmdiocbq = lpfc_sli_get_iocbq(phba);
rxbmp = kmalloc(sizeof(struct lpfc_dmabuf), GFP_KERNEL);
if (rxbmp != NULL) {
struct lpfc_vport *vport = shost_priv(fc_bsg_to_shost(job));
struct lpfc_hba *phba = vport->phba;
struct lpfc_iocbq *cmdiocb;
- struct lpfc_sli_ring *pring = &phba->sli.ring[LPFC_ELS_RING];
+ struct lpfc_sli_ring *pring;
struct bsg_job_data *dd_data;
unsigned long flags;
int rc = 0;
LIST_HEAD(completions);
struct lpfc_iocbq *check_iocb, *next_iocb;
+ pring = lpfc_phba_elsring(phba);
+
/* if job's driver data is NULL, the command completed or is in the
* the process of completing. In this case, return status to request
* so the timeout is retried. This avoids double completion issues
/*******************************************************************
* This file is part of the Emulex Linux Device Driver for *
* Fibre Channel Host Bus Adapters. *
- * Copyright (C) 2010-2015 Emulex. All rights reserved. *
+ * Copyright (C) 2017 Broadcom. All Rights Reserved. The term *
+ * “Broadcom” refers to Broadcom Limited and/or its subsidiaries. *
+ * Copyright (C) 2010-2015 Emulex. All rights reserved. *
* EMULEX and SLI are trademarks of Emulex. *
- * www.emulex.com *
+ * www.broadcom.com *
* *
* This program is free software; you can redistribute it and/or *
* modify it under the terms of version 2 of the GNU General *
/*******************************************************************
* This file is part of the Emulex Linux Device Driver for *
* Fibre Channel Host Bus Adapters. *
+ * Copyright (C) 2017 Broadcom. All Rights Reserved. The term *
+ * “Broadcom” refers to Broadcom Limited and/or its subsidiaries. *
* Copyright (C) 2004-2011 Emulex. All rights reserved. *
* EMULEX and SLI are trademarks of Emulex. *
- * www.emulex.com *
+ * www.broadcom.com *
* *
* This program is free software; you can redistribute it and/or *
* modify it under the terms of version 2 of the GNU General *
/*******************************************************************
* This file is part of the Emulex Linux Device Driver for *
* Fibre Channel Host Bus Adapters. *
+ * Copyright (C) 2017 Broadcom. All Rights Reserved. The term *
+ * “Broadcom” refers to Broadcom Limited and/or its subsidiaries. *
* Copyright (C) 2004-2016 Emulex. All rights reserved. *
* EMULEX and SLI are trademarks of Emulex. *
- * www.emulex.com *
+ * www.broadcom.com *
* *
* This program is free software; you can redistribute it and/or *
* modify it under the terms of version 2 of the GNU General *
typedef int (*node_filter)(struct lpfc_nodelist *, void *);
struct fc_rport;
+struct fc_frame_header;
void lpfc_down_link(struct lpfc_hba *, LPFC_MBOXQ_t *);
void lpfc_sli_read_link_ste(struct lpfc_hba *);
void lpfc_dump_mem(struct lpfc_hba *, LPFC_MBOXQ_t *, uint16_t, uint16_t);
void lpfc_ct_unsol_event(struct lpfc_hba *, struct lpfc_sli_ring *,
struct lpfc_iocbq *);
int lpfc_ct_handle_unsol_abort(struct lpfc_hba *, struct hbq_dmabuf *);
+int lpfc_issue_gidft(struct lpfc_vport *vport);
+int lpfc_get_gidft_type(struct lpfc_vport *vport, struct lpfc_iocbq *iocbq);
int lpfc_ns_cmd(struct lpfc_vport *, int, uint8_t, uint32_t);
int lpfc_fdmi_cmd(struct lpfc_vport *, struct lpfc_nodelist *, int, uint32_t);
void lpfc_fdmi_num_disc_check(struct lpfc_vport *);
void lpfc_offline_prep(struct lpfc_hba *, int);
void lpfc_offline(struct lpfc_hba *);
void lpfc_reset_hba(struct lpfc_hba *);
+int lpfc_emptyq_wait(struct lpfc_hba *phba, struct list_head *hd,
+ spinlock_t *slock);
int lpfc_fof_queue_create(struct lpfc_hba *);
int lpfc_fof_queue_setup(struct lpfc_hba *);
irqreturn_t lpfc_sli4_fof_intr_handler(int, void *);
int lpfc_sli_setup(struct lpfc_hba *);
-int lpfc_sli_queue_setup(struct lpfc_hba *);
+int lpfc_sli4_setup(struct lpfc_hba *phba);
+void lpfc_sli_queue_init(struct lpfc_hba *phba);
+void lpfc_sli4_queue_init(struct lpfc_hba *phba);
+struct lpfc_sli_ring *lpfc_sli4_calc_ring(struct lpfc_hba *phba,
+ struct lpfc_iocbq *iocbq);
void lpfc_handle_eratt(struct lpfc_hba *);
void lpfc_handle_latt(struct lpfc_hba *);
void lpfc_init_vpi(struct lpfc_hba *, struct lpfcMboxq *, uint16_t);
void lpfc_unreg_vfi(struct lpfcMboxq *, struct lpfc_vport *);
void lpfc_reg_fcfi(struct lpfc_hba *, struct lpfcMboxq *);
+void lpfc_reg_fcfi_mrq(struct lpfc_hba *phba, struct lpfcMboxq *mbox, int mode);
void lpfc_unreg_fcfi(struct lpfcMboxq *, uint16_t);
void lpfc_resume_rpi(struct lpfcMboxq *, struct lpfc_nodelist *);
int lpfc_check_pending_fcoe_event(struct lpfc_hba *, uint8_t);
void lpfc_els_hbq_free(struct lpfc_hba *, struct hbq_dmabuf *);
struct hbq_dmabuf *lpfc_sli4_rb_alloc(struct lpfc_hba *);
void lpfc_sli4_rb_free(struct lpfc_hba *, struct hbq_dmabuf *);
+struct rqb_dmabuf *lpfc_sli4_nvmet_alloc(struct lpfc_hba *phba);
+void lpfc_sli4_nvmet_free(struct lpfc_hba *phba, struct rqb_dmabuf *dmab);
void lpfc_sli4_build_dflt_fcf_record(struct lpfc_hba *, struct fcf_record *,
uint16_t);
+int lpfc_sli4_rq_put(struct lpfc_queue *hq, struct lpfc_queue *dq,
+ struct lpfc_rqe *hrqe, struct lpfc_rqe *drqe);
+int lpfc_post_rq_buffer(struct lpfc_hba *phba, struct lpfc_queue *hq,
+ struct lpfc_queue *dq, int count);
+int lpfc_free_rq_buffer(struct lpfc_hba *phba, struct lpfc_queue *hq);
void lpfc_unregister_fcf(struct lpfc_hba *);
void lpfc_unregister_fcf_rescan(struct lpfc_hba *);
void lpfc_unregister_unused_fcf(struct lpfc_hba *);
void lpfc_sli4_unreg_rpi_cmpl_clr(struct lpfc_hba *, LPFC_MBOXQ_t *);
int lpfc_sli_issue_iocb(struct lpfc_hba *, uint32_t,
struct lpfc_iocbq *, uint32_t);
+int lpfc_sli4_issue_wqe(struct lpfc_hba *phba, uint32_t rnum,
+ struct lpfc_iocbq *iocbq);
+struct lpfc_sglq *__lpfc_clear_active_sglq(struct lpfc_hba *phba, uint16_t xri);
+struct lpfc_sglq *__lpfc_sli_get_nvmet_sglq(struct lpfc_hba *phba,
+ struct lpfc_iocbq *piocbq);
void lpfc_sli_pcimem_bcopy(void *, void *, uint32_t);
void lpfc_sli_bemem_bcopy(void *, void *, uint32_t);
void lpfc_sli_abort_iocb_ring(struct lpfc_hba *, struct lpfc_sli_ring *);
void *lpfc_mbuf_alloc(struct lpfc_hba *, int, dma_addr_t *);
void __lpfc_mbuf_free(struct lpfc_hba *, void *, dma_addr_t);
void lpfc_mbuf_free(struct lpfc_hba *, void *, dma_addr_t);
+void *lpfc_nvmet_buf_alloc(struct lpfc_hba *phba, int flags,
+ dma_addr_t *handle);
+void lpfc_nvmet_buf_free(struct lpfc_hba *phba, void *virtp, dma_addr_t dma);
void lpfc_in_buf_free(struct lpfc_hba *, struct lpfc_dmabuf *);
+void lpfc_rq_buf_free(struct lpfc_hba *phba, struct lpfc_dmabuf *mp);
+
/* Function prototypes. */
const char* lpfc_info(struct Scsi_Host *);
int lpfc_scan_finished(struct Scsi_Host *, unsigned long);
extern struct device_attribute *lpfc_vport_attrs[];
extern struct scsi_host_template lpfc_template;
extern struct scsi_host_template lpfc_template_s3;
+extern struct scsi_host_template lpfc_template_nvme;
extern struct scsi_host_template lpfc_vport_template;
extern struct fc_function_template lpfc_transport_functions;
extern struct fc_function_template lpfc_vport_transport_functions;
extern void lpfc_debugfs_initialize(struct lpfc_vport *);
extern void lpfc_debugfs_terminate(struct lpfc_vport *);
extern void lpfc_debugfs_disc_trc(struct lpfc_vport *, int, char *, uint32_t,
- uint32_t, uint32_t);
+ uint32_t, uint32_t);
extern void lpfc_debugfs_slow_ring_trc(struct lpfc_hba *, char *, uint32_t,
- uint32_t, uint32_t);
+ uint32_t, uint32_t);
+extern void lpfc_debugfs_nvme_trc(struct lpfc_hba *phba, char *fmt,
+ uint16_t data1, uint16_t data2, uint32_t data3);
extern struct lpfc_hbq_init *lpfc_hbq_defs[];
/* SLI4 if_type 2 externs. */
int lpfc_selective_reset(struct lpfc_hba *);
int lpfc_sli4_read_config(struct lpfc_hba *);
void lpfc_sli4_node_prep(struct lpfc_hba *);
-int lpfc_sli4_xri_sgl_update(struct lpfc_hba *);
+int lpfc_sli4_els_sgl_update(struct lpfc_hba *phba);
+int lpfc_sli4_nvmet_sgl_update(struct lpfc_hba *phba);
+int lpfc_sli4_scsi_sgl_update(struct lpfc_hba *phba);
+int lpfc_sli4_nvme_sgl_update(struct lpfc_hba *phba);
void lpfc_free_sgl_list(struct lpfc_hba *, struct list_head *);
uint32_t lpfc_sli_port_speed_get(struct lpfc_hba *);
int lpfc_sli4_request_firmware_update(struct lpfc_hba *, uint8_t);
uint32_t *, uint32_t *);
int lpfc_sli4_dump_page_a0(struct lpfc_hba *phba, struct lpfcMboxq *mbox);
void lpfc_mbx_cmpl_rdp_page_a0(struct lpfc_hba *phba, LPFC_MBOXQ_t *pmb);
+
+/* NVME interfaces. */
+void lpfc_nvme_unregister_port(struct lpfc_vport *vport,
+ struct lpfc_nodelist *ndlp);
+int lpfc_nvme_register_port(struct lpfc_vport *vport,
+ struct lpfc_nodelist *ndlp);
+int lpfc_nvme_create_localport(struct lpfc_vport *vport);
+void lpfc_nvme_destroy_localport(struct lpfc_vport *vport);
+void lpfc_nvme_update_localport(struct lpfc_vport *vport);
+int lpfc_nvmet_create_targetport(struct lpfc_hba *phba);
+int lpfc_nvmet_update_targetport(struct lpfc_hba *phba);
+void lpfc_nvmet_destroy_targetport(struct lpfc_hba *phba);
+void lpfc_nvmet_unsol_ls_event(struct lpfc_hba *phba,
+ struct lpfc_sli_ring *pring, struct lpfc_iocbq *piocb);
+void lpfc_nvmet_unsol_fcp_event(struct lpfc_hba *phba,
+ struct lpfc_sli_ring *pring,
+ struct rqb_dmabuf *nvmebuf, uint64_t isr_ts);
+void lpfc_nvme_mod_param_dep(struct lpfc_hba *phba);
+void lpfc_nvme_abort_fcreq_cmpl(struct lpfc_hba *phba,
+ struct lpfc_iocbq *cmdiocb,
+ struct lpfc_wcqe_complete *abts_cmpl);
+extern int lpfc_enable_nvmet_cnt;
+extern unsigned long long lpfc_enable_nvmet[];
/*******************************************************************
* This file is part of the Emulex Linux Device Driver for *
* Fibre Channel Host Bus Adapters. *
+ * Copyright (C) 2017 Broadcom. All Rights Reserved. The term *
+ * “Broadcom” refers to Broadcom Limited and/or its subsidiaries. *
* Copyright (C) 2004-2016 Emulex. All rights reserved. *
* EMULEX and SLI are trademarks of Emulex. *
- * www.emulex.com *
+ * www.broadcom.com *
* *
* This program is free software; you can redistribute it and/or *
* modify it under the terms of version 2 of the GNU General *
#include "lpfc_sli4.h"
#include "lpfc_nl.h"
#include "lpfc_disc.h"
-#include "lpfc_scsi.h"
#include "lpfc.h"
+#include "lpfc_scsi.h"
+#include "lpfc_nvme.h"
#include "lpfc_logmsg.h"
#include "lpfc_crtn.h"
#include "lpfc_version.h"
return NULL;
}
+static void
+lpfc_prep_node_fc4type(struct lpfc_vport *vport, uint32_t Did, uint8_t fc4_type)
+{
+ struct lpfc_nodelist *ndlp;
+
+ if ((vport->port_type != LPFC_NPIV_PORT) ||
+ !(vport->ct_flags & FC_CT_RFF_ID) || !vport->cfg_restrict_login) {
+
+ ndlp = lpfc_setup_disc_node(vport, Did);
+
+ if (ndlp && NLP_CHK_NODE_ACT(ndlp)) {
+ lpfc_debugfs_disc_trc(vport, LPFC_DISC_TRC_CT,
+ "Parse GID_FTrsp: did:x%x flg:x%x x%x",
+ Did, ndlp->nlp_flag, vport->fc_flag);
+
+ /* By default, the driver expects to support FCP FC4 */
+ if (fc4_type == FC_TYPE_FCP)
+ ndlp->nlp_fc4_type |= NLP_FC4_FCP;
+
+ if (fc4_type == FC_TYPE_NVME)
+ ndlp->nlp_fc4_type |= NLP_FC4_NVME;
+
+ lpfc_printf_vlog(vport, KERN_INFO, LOG_DISCOVERY,
+ "0238 Process x%06x NameServer Rsp "
+ "Data: x%x x%x x%x x%x\n", Did,
+ ndlp->nlp_flag, ndlp->nlp_fc4_type,
+ vport->fc_flag,
+ vport->fc_rscn_id_cnt);
+ } else {
+ lpfc_debugfs_disc_trc(vport, LPFC_DISC_TRC_CT,
+ "Skip1 GID_FTrsp: did:x%x flg:x%x cnt:%d",
+ Did, vport->fc_flag, vport->fc_rscn_id_cnt);
+
+ lpfc_printf_vlog(vport, KERN_INFO, LOG_DISCOVERY,
+ "0239 Skip x%06x NameServer Rsp "
+ "Data: x%x x%x\n", Did,
+ vport->fc_flag,
+ vport->fc_rscn_id_cnt);
+ }
+ } else {
+ if (!(vport->fc_flag & FC_RSCN_MODE) ||
+ lpfc_rscn_payload_check(vport, Did)) {
+ lpfc_debugfs_disc_trc(vport, LPFC_DISC_TRC_CT,
+ "Query GID_FTrsp: did:x%x flg:x%x cnt:%d",
+ Did, vport->fc_flag, vport->fc_rscn_id_cnt);
+
+ /*
+ * This NPortID was previously a FCP target,
+ * Don't even bother to send GFF_ID.
+ */
+ ndlp = lpfc_findnode_did(vport, Did);
+ if (ndlp && NLP_CHK_NODE_ACT(ndlp))
+ ndlp->nlp_fc4_type = fc4_type;
+
+ if (ndlp && NLP_CHK_NODE_ACT(ndlp)) {
+ ndlp->nlp_fc4_type = fc4_type;
+
+ if (ndlp->nlp_type & NLP_FCP_TARGET)
+ lpfc_setup_disc_node(vport, Did);
+
+ else if (lpfc_ns_cmd(vport, SLI_CTNS_GFF_ID,
+ 0, Did) == 0)
+ vport->num_disc_nodes++;
+
+ else
+ lpfc_setup_disc_node(vport, Did);
+ }
+ } else {
+ lpfc_debugfs_disc_trc(vport, LPFC_DISC_TRC_CT,
+ "Skip2 GID_FTrsp: did:x%x flg:x%x cnt:%d",
+ Did, vport->fc_flag, vport->fc_rscn_id_cnt);
+
+ lpfc_printf_vlog(vport, KERN_INFO, LOG_DISCOVERY,
+ "0245 Skip x%06x NameServer Rsp "
+ "Data: x%x x%x\n", Did,
+ vport->fc_flag,
+ vport->fc_rscn_id_cnt);
+ }
+ }
+}
+
static int
-lpfc_ns_rsp(struct lpfc_vport *vport, struct lpfc_dmabuf *mp, uint32_t Size)
+lpfc_ns_rsp(struct lpfc_vport *vport, struct lpfc_dmabuf *mp, uint8_t fc4_type,
+ uint32_t Size)
{
struct lpfc_hba *phba = vport->phba;
struct lpfc_sli_ct_request *Response =
*/
if ((Did != vport->fc_myDID) &&
((lpfc_find_vport_by_did(phba, Did) == NULL) ||
- vport->cfg_peer_port_login)) {
- if ((vport->port_type != LPFC_NPIV_PORT) ||
- (!(vport->ct_flags & FC_CT_RFF_ID)) ||
- (!vport->cfg_restrict_login)) {
- ndlp = lpfc_setup_disc_node(vport, Did);
- if (ndlp && NLP_CHK_NODE_ACT(ndlp)) {
- lpfc_debugfs_disc_trc(vport,
- LPFC_DISC_TRC_CT,
- "Parse GID_FTrsp: "
- "did:x%x flg:x%x x%x",
- Did, ndlp->nlp_flag,
- vport->fc_flag);
-
- lpfc_printf_vlog(vport,
- KERN_INFO,
- LOG_DISCOVERY,
- "0238 Process "
- "x%x NameServer Rsp"
- "Data: x%x x%x x%x\n",
- Did, ndlp->nlp_flag,
- vport->fc_flag,
- vport->fc_rscn_id_cnt);
- } else {
- lpfc_debugfs_disc_trc(vport,
- LPFC_DISC_TRC_CT,
- "Skip1 GID_FTrsp: "
- "did:x%x flg:x%x cnt:%d",
- Did, vport->fc_flag,
- vport->fc_rscn_id_cnt);
-
- lpfc_printf_vlog(vport,
- KERN_INFO,
- LOG_DISCOVERY,
- "0239 Skip x%x "
- "NameServer Rsp Data: "
- "x%x x%x\n",
- Did, vport->fc_flag,
- vport->fc_rscn_id_cnt);
- }
-
- } else {
- if (!(vport->fc_flag & FC_RSCN_MODE) ||
- (lpfc_rscn_payload_check(vport, Did))) {
- lpfc_debugfs_disc_trc(vport,
- LPFC_DISC_TRC_CT,
- "Query GID_FTrsp: "
- "did:x%x flg:x%x cnt:%d",
- Did, vport->fc_flag,
- vport->fc_rscn_id_cnt);
-
- /* This NPortID was previously
- * a FCP target, * Don't even
- * bother to send GFF_ID.
- */
- ndlp = lpfc_findnode_did(vport,
- Did);
- if (ndlp &&
- NLP_CHK_NODE_ACT(ndlp)
- && (ndlp->nlp_type &
- NLP_FCP_TARGET))
- lpfc_setup_disc_node
- (vport, Did);
- else if (lpfc_ns_cmd(vport,
- SLI_CTNS_GFF_ID,
- 0, Did) == 0)
- vport->num_disc_nodes++;
- else
- lpfc_setup_disc_node
- (vport, Did);
- }
- else {
- lpfc_debugfs_disc_trc(vport,
- LPFC_DISC_TRC_CT,
- "Skip2 GID_FTrsp: "
- "did:x%x flg:x%x cnt:%d",
- Did, vport->fc_flag,
- vport->fc_rscn_id_cnt);
-
- lpfc_printf_vlog(vport,
- KERN_INFO,
- LOG_DISCOVERY,
- "0245 Skip x%x "
- "NameServer Rsp Data: "
- "x%x x%x\n",
- Did, vport->fc_flag,
- vport->fc_rscn_id_cnt);
- }
- }
- }
+ vport->cfg_peer_port_login))
+ lpfc_prep_node_fc4type(vport, Did, fc4_type);
+
if (CTentry & (cpu_to_be32(SLI_CT_LAST_ENTRY)))
goto nsout1;
+
Cnt -= sizeof(uint32_t);
}
ctptr = NULL;
struct Scsi_Host *shost = lpfc_shost_from_vport(vport);
IOCB_t *irsp;
struct lpfc_dmabuf *outp;
+ struct lpfc_dmabuf *inp;
struct lpfc_sli_ct_request *CTrsp;
+ struct lpfc_sli_ct_request *CTreq;
struct lpfc_nodelist *ndlp;
- int rc;
+ int rc, type;
/* First save ndlp, before we overwrite it */
ndlp = cmdiocb->context_un.ndlp;
/* we pass cmdiocb to state machine which needs rspiocb as well */
cmdiocb->context_un.rsp_iocb = rspiocb;
-
+ inp = (struct lpfc_dmabuf *) cmdiocb->context1;
outp = (struct lpfc_dmabuf *) cmdiocb->context2;
irsp = &rspiocb->iocb;
IOERR_NO_RESOURCES)
vport->fc_ns_retry++;
+ type = lpfc_get_gidft_type(vport, cmdiocb);
+ if (type == 0)
+ goto out;
+
/* CT command is being retried */
+ vport->gidft_inp--;
rc = lpfc_ns_cmd(vport, SLI_CTNS_GID_FT,
- vport->fc_ns_retry, 0);
+ vport->fc_ns_retry, type);
if (rc == 0)
goto out;
}
irsp->ulpStatus, vport->fc_ns_retry);
} else {
/* Good status, continue checking */
+ CTreq = (struct lpfc_sli_ct_request *) inp->virt;
CTrsp = (struct lpfc_sli_ct_request *) outp->virt;
if (CTrsp->CommandResponse.bits.CmdRsp ==
cpu_to_be16(SLI_CT_RESPONSE_FS_ACC)) {
lpfc_printf_vlog(vport, KERN_INFO, LOG_DISCOVERY,
- "0208 NameServer Rsp Data: x%x\n",
- vport->fc_flag);
- lpfc_ns_rsp(vport, outp,
+ "0208 NameServer Rsp Data: x%x x%x\n",
+ vport->fc_flag,
+ CTreq->un.gid.Fc4Type);
+
+ lpfc_ns_rsp(vport,
+ outp,
+ CTreq->un.gid.Fc4Type,
(uint32_t) (irsp->un.genreq64.bdl.bdeSize));
} else if (CTrsp->CommandResponse.bits.CmdRsp ==
be16_to_cpu(SLI_CT_RESPONSE_FS_RJT)) {
(uint32_t) CTrsp->ReasonCode,
(uint32_t) CTrsp->Explanation);
}
+ vport->gidft_inp--;
}
/* Link up / RSCN discovery */
- if (vport->num_disc_nodes == 0) {
+ if ((vport->num_disc_nodes == 0) &&
+ (vport->gidft_inp == 0)) {
/*
* The driver has cycled through all Nports in the RSCN payload.
* Complete the handling by cleaning up and marking the
return;
}
+static void
+lpfc_cmpl_ct_cmd_gft_id(struct lpfc_hba *phba, struct lpfc_iocbq *cmdiocb,
+ struct lpfc_iocbq *rspiocb)
+{
+ struct lpfc_vport *vport = cmdiocb->vport;
+ IOCB_t *irsp = &rspiocb->iocb;
+ struct lpfc_dmabuf *inp = (struct lpfc_dmabuf *)cmdiocb->context1;
+ struct lpfc_dmabuf *outp = (struct lpfc_dmabuf *)cmdiocb->context2;
+ struct lpfc_sli_ct_request *CTrsp;
+ int did;
+ struct lpfc_nodelist *ndlp;
+ uint32_t fc4_data_0, fc4_data_1;
+
+ did = ((struct lpfc_sli_ct_request *)inp->virt)->un.gft.PortId;
+ did = be32_to_cpu(did);
+
+ lpfc_debugfs_disc_trc(vport, LPFC_DISC_TRC_CT,
+ "GFT_ID cmpl: status:x%x/x%x did:x%x",
+ irsp->ulpStatus, irsp->un.ulpWord[4], did);
+
+ if (irsp->ulpStatus == IOSTAT_SUCCESS) {
+ /* Good status, continue checking */
+ CTrsp = (struct lpfc_sli_ct_request *)outp->virt;
+ fc4_data_0 = be32_to_cpu(CTrsp->un.gft_acc.fc4_types[0]);
+ fc4_data_1 = be32_to_cpu(CTrsp->un.gft_acc.fc4_types[1]);
+ lpfc_printf_vlog(vport, KERN_ERR, LOG_DISCOVERY,
+ "3062 DID x%06x GFT Wd0 x%08x Wd1 x%08x\n",
+ did, fc4_data_0, fc4_data_1);
+
+ ndlp = lpfc_findnode_did(vport, did);
+ if (ndlp) {
+ /* The bitmask value for FCP and NVME FCP types is
+ * the same because they are 32 bits distant from
+ * each other in word0 and word0.
+ */
+ if (fc4_data_0 & LPFC_FC4_TYPE_BITMASK)
+ ndlp->nlp_fc4_type |= NLP_FC4_FCP;
+ if (fc4_data_1 & LPFC_FC4_TYPE_BITMASK)
+ ndlp->nlp_fc4_type |= NLP_FC4_NVME;
+ lpfc_printf_vlog(vport, KERN_ERR, LOG_DISCOVERY,
+ "3064 Setting ndlp %p, DID x%06x with "
+ "FC4 x%08x, Data: x%08x x%08x\n",
+ ndlp, did, ndlp->nlp_fc4_type,
+ FC_TYPE_FCP, FC_TYPE_NVME);
+ }
+ ndlp->nlp_prev_state = NLP_STE_REG_LOGIN_ISSUE;
+ lpfc_nlp_set_state(vport, ndlp, NLP_STE_PRLI_ISSUE);
+ lpfc_issue_els_prli(vport, ndlp, 0);
+ } else
+ lpfc_printf_vlog(vport, KERN_ERR, LOG_DISCOVERY,
+ "3065 GFT_ID failed x%08x\n", irsp->ulpStatus);
+
+ lpfc_ct_free_iocb(phba, cmdiocb);
+}
static void
lpfc_cmpl_ct(struct lpfc_hba *phba, struct lpfc_iocbq *cmdiocb,
return;
}
+/*
+ * Although the symbolic port name is thought to be an integer
+ * as of January 18, 2016, leave it as a string until more of
+ * the record state becomes defined.
+ */
int
lpfc_vport_symbolic_port_name(struct lpfc_vport *vport, char *symbol,
size_t size)
{
int n;
- uint8_t *wwn = vport->phba->wwpn;
- n = snprintf(symbol, size,
- "Emulex PPN-%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x",
- wwn[0], wwn[1], wwn[2], wwn[3],
- wwn[4], wwn[5], wwn[6], wwn[7]);
-
- if (vport->port_type == LPFC_PHYSICAL_PORT)
- return n;
-
- if (n < size)
- n += snprintf(symbol + n, size - n, " VPort-%d", vport->vpi);
-
- if (n < size &&
- strlen(vport->fc_vport->symbolic_name))
- n += snprintf(symbol + n, size - n, " VName-%s",
- vport->fc_vport->symbolic_name);
+ /*
+ * Use the lpfc board number as the Symbolic Port
+ * Name object. NPIV is not in play so this integer
+ * value is sufficient and unique per FC-ID.
+ */
+ n = snprintf(symbol, size, "%d", vport->phba->brd_no);
return n;
}
+
int
lpfc_vport_symbolic_node_name(struct lpfc_vport *vport, char *symbol,
size_t size)
lpfc_decode_firmware_rev(vport->phba, fwrev, 0);
n = snprintf(symbol, size, "Emulex %s", vport->phba->ModelName);
-
if (size < n)
return n;
- n += snprintf(symbol + n, size - n, " FV%s", fwrev);
+ n += snprintf(symbol + n, size - n, " FV%s", fwrev);
if (size < n)
return n;
- n += snprintf(symbol + n, size - n, " DV%s", lpfc_release_version);
+ n += snprintf(symbol + n, size - n, " DV%s.",
+ lpfc_release_version);
if (size < n)
return n;
- n += snprintf(symbol + n, size - n, " HN:%s", init_utsname()->nodename);
- /* Note :- OS name is "Linux" */
+ n += snprintf(symbol + n, size - n, " HN:%s.",
+ init_utsname()->nodename);
if (size < n)
return n;
- n += snprintf(symbol + n, size - n, " OS:%s", init_utsname()->sysname);
+ /* Note :- OS name is "Linux" */
+ n += snprintf(symbol + n, size - n, " OS:%s\n",
+ init_utsname()->sysname);
return n;
}
return cnt;
}
+/*
+ * This routine will return the FC4 Type associated with the CT
+ * GID_FT command.
+ */
+int
+lpfc_get_gidft_type(struct lpfc_vport *vport, struct lpfc_iocbq *cmdiocb)
+{
+ struct lpfc_sli_ct_request *CtReq;
+ struct lpfc_dmabuf *mp;
+ uint32_t type;
+
+ mp = cmdiocb->context1;
+ if (mp == NULL)
+ return 0;
+ CtReq = (struct lpfc_sli_ct_request *)mp->virt;
+ type = (uint32_t)CtReq->un.gid.Fc4Type;
+ if ((type != SLI_CTPT_FCP) && (type != SLI_CTPT_NVME))
+ return 0;
+ return type;
+}
+
/*
* lpfc_ns_cmd
* Description:
/* NameServer Req */
lpfc_printf_vlog(vport, KERN_INFO ,LOG_DISCOVERY,
- "0236 NameServer Req Data: x%x x%x x%x\n",
- cmdcode, vport->fc_flag, vport->fc_rscn_id_cnt);
+ "0236 NameServer Req Data: x%x x%x x%x x%x\n",
+ cmdcode, vport->fc_flag, vport->fc_rscn_id_cnt,
+ context);
bpl = (struct ulp_bde64 *) bmp->virt;
memset(bpl, 0, sizeof(struct ulp_bde64));
bpl->tus.f.bdeSize = GID_REQUEST_SZ;
else if (cmdcode == SLI_CTNS_GFF_ID)
bpl->tus.f.bdeSize = GFF_REQUEST_SZ;
+ else if (cmdcode == SLI_CTNS_GFT_ID)
+ bpl->tus.f.bdeSize = GFT_REQUEST_SZ;
else if (cmdcode == SLI_CTNS_RFT_ID)
bpl->tus.f.bdeSize = RFT_REQUEST_SZ;
else if (cmdcode == SLI_CTNS_RNN_ID)
case SLI_CTNS_GID_FT:
CtReq->CommandResponse.bits.CmdRsp =
cpu_to_be16(SLI_CTNS_GID_FT);
- CtReq->un.gid.Fc4Type = SLI_CTPT_FCP;
+ CtReq->un.gid.Fc4Type = context;
+
if (vport->port_state < LPFC_NS_QRY)
vport->port_state = LPFC_NS_QRY;
lpfc_set_disctmo(vport);
cmpl = lpfc_cmpl_ct_cmd_gff_id;
break;
+ case SLI_CTNS_GFT_ID:
+ CtReq->CommandResponse.bits.CmdRsp =
+ cpu_to_be16(SLI_CTNS_GFT_ID);
+ CtReq->un.gft.PortId = cpu_to_be32(context);
+ cmpl = lpfc_cmpl_ct_cmd_gft_id;
+ break;
+
case SLI_CTNS_RFT_ID:
vport->ct_flags &= ~FC_CT_RFT_ID;
CtReq->CommandResponse.bits.CmdRsp =
cpu_to_be16(SLI_CTNS_RFT_ID);
CtReq->un.rft.PortId = cpu_to_be32(vport->fc_myDID);
- CtReq->un.rft.fcpReg = 1;
+
+ /* Register FC4 FCP type if enabled. */
+ if ((phba->cfg_enable_fc4_type == LPFC_ENABLE_BOTH) ||
+ (phba->cfg_enable_fc4_type == LPFC_ENABLE_FCP))
+ CtReq->un.rft.fcpReg = 1;
+
+ /* Register NVME type if enabled. Defined LE and swapped.
+ * rsvd[0] is used as word1 because of the hard-coded
+ * word0 usage in the ct_request data structure.
+ */
+ if ((phba->cfg_enable_fc4_type == LPFC_ENABLE_BOTH) ||
+ (phba->cfg_enable_fc4_type == LPFC_ENABLE_NVME))
+ CtReq->un.rft.rsvd[0] = cpu_to_be32(0x00000100);
+
cmpl = lpfc_cmpl_ct_cmd_rft_id;
break;
cpu_to_be16(SLI_CTNS_RFF_ID);
CtReq->un.rff.PortId = cpu_to_be32(vport->fc_myDID);
CtReq->un.rff.fbits = FC4_FEATURE_INIT;
- CtReq->un.rff.type_code = FC_TYPE_FCP;
+
+ /* The driver always supports FC_TYPE_FCP. However, the
+ * caller can specify NVME (type x28) as well. But only
+ * these that FC4 type is supported.
+ */
+ if (((phba->cfg_enable_fc4_type == LPFC_ENABLE_BOTH) ||
+ (phba->cfg_enable_fc4_type == LPFC_ENABLE_NVME)) &&
+ (context == FC_TYPE_NVME)) {
+ if ((vport == phba->pport) && phba->nvmet_support) {
+ CtReq->un.rff.fbits = (FC4_FEATURE_TARGET |
+ FC4_FEATURE_NVME_DISC);
+ lpfc_nvmet_update_targetport(phba);
+ } else {
+ lpfc_nvme_update_localport(vport);
+ }
+ CtReq->un.rff.type_code = context;
+
+ } else if (((phba->cfg_enable_fc4_type == LPFC_ENABLE_BOTH) ||
+ (phba->cfg_enable_fc4_type == LPFC_ENABLE_FCP)) &&
+ (context == FC_TYPE_FCP))
+ CtReq->un.rff.type_code = context;
+
+ else
+ goto ns_cmd_free_bmpvirt;
+
cmpl = lpfc_cmpl_ct_cmd_rff_id;
break;
}
*/
lpfc_nlp_put(ndlp);
+ns_cmd_free_bmpvirt:
lpfc_mbuf_free(phba, bmp->virt, bmp->phys);
ns_cmd_free_bmp:
kfree(bmp);
/*******************************************************************
* This file is part of the Emulex Linux Device Driver for *
* Fibre Channel Host Bus Adapters. *
+ * Copyright (C) 2017 Broadcom. All Rights Reserved. The term *
+ * “Broadcom” refers to Broadcom Limited and/or its subsidiaries. *
* Copyright (C) 2007-2015 Emulex. All rights reserved. *
* EMULEX and SLI are trademarks of Emulex. *
- * www.emulex.com *
+ * www.broadcom.com *
* *
* This program is free software; you can redistribute it and/or *
* modify it under the terms of version 2 of the GNU General *
#include <scsi/scsi_device.h>
#include <scsi/scsi_host.h>
#include <scsi/scsi_transport_fc.h>
+#include <scsi/fc/fc_fs.h>
+
+#include <linux/nvme-fc-driver.h>
#include "lpfc_hw4.h"
#include "lpfc_hw.h"
#include "lpfc_sli4.h"
#include "lpfc_nl.h"
#include "lpfc_disc.h"
-#include "lpfc_scsi.h"
#include "lpfc.h"
+#include "lpfc_scsi.h"
+#include "lpfc_nvme.h"
+#include "lpfc_nvmet.h"
#include "lpfc_logmsg.h"
#include "lpfc_crtn.h"
#include "lpfc_vport.h"
MODULE_PARM_DESC(lpfc_debugfs_max_slow_ring_trc,
"Set debugfs slow ring trace depth");
+/* This MUST be a power of 2 */
+static int lpfc_debugfs_max_nvmeio_trc;
+module_param(lpfc_debugfs_max_nvmeio_trc, int, 0444);
+MODULE_PARM_DESC(lpfc_debugfs_max_nvmeio_trc,
+ "Set debugfs NVME IO trace depth");
+
static int lpfc_debugfs_mask_disc_trc;
module_param(lpfc_debugfs_mask_disc_trc, int, S_IRUGO);
MODULE_PARM_DESC(lpfc_debugfs_mask_disc_trc,
off += (8 * sizeof(uint32_t));
}
- for (i = 0; i < 4; i++) {
- pgpp = &phba->port_gp[i];
- pring = &psli->ring[i];
- len += snprintf(buf+len, size-len,
- "Ring %d: CMD GetInx:%d (Max:%d Next:%d "
- "Local:%d flg:x%x) RSP PutInx:%d Max:%d\n",
- i, pgpp->cmdGetInx, pring->sli.sli3.numCiocb,
- pring->sli.sli3.next_cmdidx,
- pring->sli.sli3.local_getidx,
- pring->flag, pgpp->rspPutInx,
- pring->sli.sli3.numRiocb);
- }
-
if (phba->sli_rev <= LPFC_SLI_REV3) {
+ for (i = 0; i < 4; i++) {
+ pgpp = &phba->port_gp[i];
+ pring = &psli->sli3_ring[i];
+ len += snprintf(buf+len, size-len,
+ "Ring %d: CMD GetInx:%d "
+ "(Max:%d Next:%d "
+ "Local:%d flg:x%x) "
+ "RSP PutInx:%d Max:%d\n",
+ i, pgpp->cmdGetInx,
+ pring->sli.sli3.numCiocb,
+ pring->sli.sli3.next_cmdidx,
+ pring->sli.sli3.local_getidx,
+ pring->flag, pgpp->rspPutInx,
+ pring->sli.sli3.numRiocb);
+ }
+
word0 = readl(phba->HAregaddr);
word1 = readl(phba->CAregaddr);
word2 = readl(phba->HSregaddr);
int len = 0;
int cnt;
struct Scsi_Host *shost = lpfc_shost_from_vport(vport);
+ struct lpfc_hba *phba = vport->phba;
struct lpfc_nodelist *ndlp;
- unsigned char *statep, *name;
+ unsigned char *statep;
+ struct nvme_fc_local_port *localport;
+ struct lpfc_nvme_lport *lport;
+ struct lpfc_nvme_rport *rport;
+ struct lpfc_nvmet_tgtport *tgtp;
+ struct nvme_fc_remote_port *nrport;
cnt = (LPFC_NODELIST_SIZE / LPFC_NODELIST_ENTRY_SIZE);
+ len += snprintf(buf+len, size-len, "\nFCP Nodelist Entries ...\n");
spin_lock_irq(shost->host_lock);
list_for_each_entry(ndlp, &vport->fc_nodes, nlp_listp) {
if (!cnt) {
default:
statep = "UNKNOWN";
}
- len += snprintf(buf+len, size-len, "%s DID:x%06x ",
- statep, ndlp->nlp_DID);
- name = (unsigned char *)&ndlp->nlp_portname;
- len += snprintf(buf+len, size-len,
- "WWPN %02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x ",
- *name, *(name+1), *(name+2), *(name+3),
- *(name+4), *(name+5), *(name+6), *(name+7));
- name = (unsigned char *)&ndlp->nlp_nodename;
- len += snprintf(buf+len, size-len,
- "WWNN %02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x ",
- *name, *(name+1), *(name+2), *(name+3),
- *(name+4), *(name+5), *(name+6), *(name+7));
+ len += snprintf(buf+len, size-len, "%s DID:x%06x ",
+ statep, ndlp->nlp_DID);
+ len += snprintf(buf+len, size-len,
+ "WWPN x%llx ",
+ wwn_to_u64(ndlp->nlp_portname.u.wwn));
+ len += snprintf(buf+len, size-len,
+ "WWNN x%llx ",
+ wwn_to_u64(ndlp->nlp_nodename.u.wwn));
if (ndlp->nlp_flag & NLP_RPI_REGISTERED)
- len += snprintf(buf+len, size-len, "RPI:%03d ",
- ndlp->nlp_rpi);
+ len += snprintf(buf+len, size-len, "RPI:%03d ",
+ ndlp->nlp_rpi);
else
- len += snprintf(buf+len, size-len, "RPI:none ");
+ len += snprintf(buf+len, size-len, "RPI:none ");
len += snprintf(buf+len, size-len, "flag:x%08x ",
ndlp->nlp_flag);
if (!ndlp->nlp_type)
- len += snprintf(buf+len, size-len, "UNKNOWN_TYPE ");
+ len += snprintf(buf+len, size-len, "UNKNOWN_TYPE ");
if (ndlp->nlp_type & NLP_FC_NODE)
- len += snprintf(buf+len, size-len, "FC_NODE ");
+ len += snprintf(buf+len, size-len, "FC_NODE ");
if (ndlp->nlp_type & NLP_FABRIC)
- len += snprintf(buf+len, size-len, "FABRIC ");
+ len += snprintf(buf+len, size-len, "FABRIC ");
if (ndlp->nlp_type & NLP_FCP_TARGET)
- len += snprintf(buf+len, size-len, "FCP_TGT sid:%d ",
+ len += snprintf(buf+len, size-len, "FCP_TGT sid:%d ",
ndlp->nlp_sid);
if (ndlp->nlp_type & NLP_FCP_INITIATOR)
- len += snprintf(buf+len, size-len, "FCP_INITIATOR ");
+ len += snprintf(buf+len, size-len, "FCP_INITIATOR ");
len += snprintf(buf+len, size-len, "usgmap:%x ",
ndlp->nlp_usg_map);
len += snprintf(buf+len, size-len, "refcnt:%x",
len += snprintf(buf+len, size-len, "\n");
}
spin_unlock_irq(shost->host_lock);
+
+ if (phba->nvmet_support && phba->targetport && (vport == phba->pport)) {
+ tgtp = (struct lpfc_nvmet_tgtport *)phba->targetport->private;
+ len += snprintf(buf + len, size - len,
+ "\nNVME Targetport Entry ...\n");
+
+ /* Port state is only one of two values for now. */
+ if (phba->targetport->port_id)
+ statep = "REGISTERED";
+ else
+ statep = "INIT";
+ len += snprintf(buf + len, size - len,
+ "TGT WWNN x%llx WWPN x%llx State %s\n",
+ wwn_to_u64(vport->fc_nodename.u.wwn),
+ wwn_to_u64(vport->fc_portname.u.wwn),
+ statep);
+ len += snprintf(buf + len, size - len,
+ " Targetport DID x%06x\n",
+ phba->targetport->port_id);
+ goto out_exit;
+ }
+
+ len += snprintf(buf + len, size - len,
+ "\nNVME Lport/Rport Entries ...\n");
+
+ localport = vport->localport;
+ if (!localport)
+ goto out_exit;
+
+ spin_lock_irq(shost->host_lock);
+ lport = (struct lpfc_nvme_lport *)localport->private;
+
+ /* Port state is only one of two values for now. */
+ if (localport->port_id)
+ statep = "ONLINE";
+ else
+ statep = "UNKNOWN ";
+
+ len += snprintf(buf + len, size - len,
+ "Lport DID x%06x PortState %s\n",
+ localport->port_id, statep);
+
+ len += snprintf(buf + len, size - len, "\tRport List:\n");
+ list_for_each_entry(rport, &lport->rport_list, list) {
+ /* local short-hand pointer. */
+ nrport = rport->remoteport;
+
+ /* Port state is only one of two values for now. */
+ switch (nrport->port_state) {
+ case FC_OBJSTATE_ONLINE:
+ statep = "ONLINE";
+ break;
+ case FC_OBJSTATE_UNKNOWN:
+ statep = "UNKNOWN ";
+ break;
+ default:
+ statep = "UNSUPPORTED";
+ break;
+ }
+
+ /* Tab in to show lport ownership. */
+ len += snprintf(buf + len, size - len,
+ "\t%s Port ID:x%06x ",
+ statep, nrport->port_id);
+ len += snprintf(buf + len, size - len, "WWPN x%llx ",
+ nrport->port_name);
+ len += snprintf(buf + len, size - len, "WWNN x%llx ",
+ nrport->node_name);
+ switch (nrport->port_role) {
+ case FC_PORT_ROLE_NVME_INITIATOR:
+ len += snprintf(buf + len, size - len,
+ "NVME INITIATOR ");
+ break;
+ case FC_PORT_ROLE_NVME_TARGET:
+ len += snprintf(buf + len, size - len,
+ "NVME TARGET ");
+ break;
+ case FC_PORT_ROLE_NVME_DISCOVERY:
+ len += snprintf(buf + len, size - len,
+ "NVME DISCOVERY ");
+ break;
+ default:
+ len += snprintf(buf + len, size - len,
+ "UNKNOWN ROLE x%x",
+ nrport->port_role);
+ break;
+ }
+
+ /* Terminate the string. */
+ len += snprintf(buf + len, size - len, "\n");
+ }
+
+ spin_unlock_irq(shost->host_lock);
+ out_exit:
+ return len;
+}
+
+/**
+ * lpfc_debugfs_nvmestat_data - Dump target node list to a buffer
+ * @vport: The vport to gather target node info from.
+ * @buf: The buffer to dump log into.
+ * @size: The maximum amount of data to process.
+ *
+ * Description:
+ * This routine dumps the NVME statistics associated with @vport
+ *
+ * Return Value:
+ * This routine returns the amount of bytes that were dumped into @buf and will
+ * not exceed @size.
+ **/
+static int
+lpfc_debugfs_nvmestat_data(struct lpfc_vport *vport, char *buf, int size)
+{
+ struct lpfc_hba *phba = vport->phba;
+ struct lpfc_nvmet_tgtport *tgtp;
+ int len = 0;
+
+ if (phba->nvmet_support) {
+ if (!phba->targetport)
+ return len;
+ tgtp = (struct lpfc_nvmet_tgtport *)phba->targetport->private;
+ len += snprintf(buf+len, size-len,
+ "\nNVME Targetport Statistics\n");
+
+ len += snprintf(buf+len, size-len,
+ "LS: Rcv %08x Drop %08x Abort %08x\n",
+ atomic_read(&tgtp->rcv_ls_req_in),
+ atomic_read(&tgtp->rcv_ls_req_drop),
+ atomic_read(&tgtp->xmt_ls_abort));
+ if (atomic_read(&tgtp->rcv_ls_req_in) !=
+ atomic_read(&tgtp->rcv_ls_req_out)) {
+ len += snprintf(buf+len, size-len,
+ "Rcv LS: in %08x != out %08x\n",
+ atomic_read(&tgtp->rcv_ls_req_in),
+ atomic_read(&tgtp->rcv_ls_req_out));
+ }
+
+ len += snprintf(buf+len, size-len,
+ "LS: Xmt %08x Drop %08x Cmpl %08x Err %08x\n",
+ atomic_read(&tgtp->xmt_ls_rsp),
+ atomic_read(&tgtp->xmt_ls_drop),
+ atomic_read(&tgtp->xmt_ls_rsp_cmpl),
+ atomic_read(&tgtp->xmt_ls_rsp_error));
+
+ len += snprintf(buf+len, size-len,
+ "FCP: Rcv %08x Drop %08x\n",
+ atomic_read(&tgtp->rcv_fcp_cmd_in),
+ atomic_read(&tgtp->rcv_fcp_cmd_drop));
+
+ if (atomic_read(&tgtp->rcv_fcp_cmd_in) !=
+ atomic_read(&tgtp->rcv_fcp_cmd_out)) {
+ len += snprintf(buf+len, size-len,
+ "Rcv FCP: in %08x != out %08x\n",
+ atomic_read(&tgtp->rcv_fcp_cmd_in),
+ atomic_read(&tgtp->rcv_fcp_cmd_out));
+ }
+
+ len += snprintf(buf+len, size-len,
+ "FCP Rsp: read %08x readrsp %08x write %08x rsp %08x\n",
+ atomic_read(&tgtp->xmt_fcp_read),
+ atomic_read(&tgtp->xmt_fcp_read_rsp),
+ atomic_read(&tgtp->xmt_fcp_write),
+ atomic_read(&tgtp->xmt_fcp_rsp));
+
+ len += snprintf(buf+len, size-len,
+ "FCP Rsp: abort %08x drop %08x\n",
+ atomic_read(&tgtp->xmt_fcp_abort),
+ atomic_read(&tgtp->xmt_fcp_drop));
+
+ len += snprintf(buf+len, size-len,
+ "FCP Rsp Cmpl: %08x err %08x drop %08x\n",
+ atomic_read(&tgtp->xmt_fcp_rsp_cmpl),
+ atomic_read(&tgtp->xmt_fcp_rsp_error),
+ atomic_read(&tgtp->xmt_fcp_rsp_drop));
+
+ len += snprintf(buf+len, size-len,
+ "ABORT: Xmt %08x Err %08x Cmpl %08x",
+ atomic_read(&tgtp->xmt_abort_rsp),
+ atomic_read(&tgtp->xmt_abort_rsp_error),
+ atomic_read(&tgtp->xmt_abort_cmpl));
+
+ len += snprintf(buf+len, size-len, "\n");
+ } else {
+ if (!(phba->cfg_enable_fc4_type & LPFC_ENABLE_NVME))
+ return len;
+
+ len += snprintf(buf + len, size - len,
+ "\nNVME Lport Statistics\n");
+
+ len += snprintf(buf + len, size - len,
+ "LS: Xmt %016llx Cmpl %016llx\n",
+ phba->fc4NvmeLsRequests,
+ phba->fc4NvmeLsCmpls);
+
+ len += snprintf(buf + len, size - len,
+ "FCP: Rd %016llx Wr %016llx IO %016llx\n",
+ phba->fc4NvmeInputRequests,
+ phba->fc4NvmeOutputRequests,
+ phba->fc4NvmeControlRequests);
+
+ len += snprintf(buf + len, size - len,
+ " Cmpl %016llx\n", phba->fc4NvmeIoCmpls);
+ }
+
+ return len;
+}
+
+
+/**
+ * lpfc_debugfs_nvmektime_data - Dump target node list to a buffer
+ * @vport: The vport to gather target node info from.
+ * @buf: The buffer to dump log into.
+ * @size: The maximum amount of data to process.
+ *
+ * Description:
+ * This routine dumps the NVME statistics associated with @vport
+ *
+ * Return Value:
+ * This routine returns the amount of bytes that were dumped into @buf and will
+ * not exceed @size.
+ **/
+static int
+lpfc_debugfs_nvmektime_data(struct lpfc_vport *vport, char *buf, int size)
+{
+ struct lpfc_hba *phba = vport->phba;
+ int len = 0;
+
+ if (phba->nvmet_support == 0) {
+ /* NVME Initiator */
+ len += snprintf(buf + len, PAGE_SIZE - len,
+ "ktime %s: Total Samples: %lld\n",
+ (phba->ktime_on ? "Enabled" : "Disabled"),
+ phba->ktime_data_samples);
+ if (phba->ktime_data_samples == 0)
+ return len;
+
+ len += snprintf(
+ buf + len, PAGE_SIZE - len,
+ "Segment 1: Last NVME Cmd cmpl "
+ "done -to- Start of next NVME cnd (in driver)\n");
+ len += snprintf(
+ buf + len, PAGE_SIZE - len,
+ "avg:%08lld min:%08lld max %08lld\n",
+ div_u64(phba->ktime_seg1_total,
+ phba->ktime_data_samples),
+ phba->ktime_seg1_min,
+ phba->ktime_seg1_max);
+ len += snprintf(
+ buf + len, PAGE_SIZE - len,
+ "Segment 2: Driver start of NVME cmd "
+ "-to- Firmware WQ doorbell\n");
+ len += snprintf(
+ buf + len, PAGE_SIZE - len,
+ "avg:%08lld min:%08lld max %08lld\n",
+ div_u64(phba->ktime_seg2_total,
+ phba->ktime_data_samples),
+ phba->ktime_seg2_min,
+ phba->ktime_seg2_max);
+ len += snprintf(
+ buf + len, PAGE_SIZE - len,
+ "Segment 3: Firmware WQ doorbell -to- "
+ "MSI-X ISR cmpl\n");
+ len += snprintf(
+ buf + len, PAGE_SIZE - len,
+ "avg:%08lld min:%08lld max %08lld\n",
+ div_u64(phba->ktime_seg3_total,
+ phba->ktime_data_samples),
+ phba->ktime_seg3_min,
+ phba->ktime_seg3_max);
+ len += snprintf(
+ buf + len, PAGE_SIZE - len,
+ "Segment 4: MSI-X ISR cmpl -to- "
+ "NVME cmpl done\n");
+ len += snprintf(
+ buf + len, PAGE_SIZE - len,
+ "avg:%08lld min:%08lld max %08lld\n",
+ div_u64(phba->ktime_seg4_total,
+ phba->ktime_data_samples),
+ phba->ktime_seg4_min,
+ phba->ktime_seg4_max);
+ len += snprintf(
+ buf + len, PAGE_SIZE - len,
+ "Total IO avg time: %08lld\n",
+ div_u64(phba->ktime_seg1_total +
+ phba->ktime_seg2_total +
+ phba->ktime_seg3_total +
+ phba->ktime_seg4_total,
+ phba->ktime_data_samples));
+ return len;
+ }
+
+ /* NVME Target */
+ len += snprintf(buf + len, PAGE_SIZE-len,
+ "ktime %s: Total Samples: %lld %lld\n",
+ (phba->ktime_on ? "Enabled" : "Disabled"),
+ phba->ktime_data_samples,
+ phba->ktime_status_samples);
+ if (phba->ktime_data_samples == 0)
+ return len;
+
+ len += snprintf(buf + len, PAGE_SIZE-len,
+ "Segment 1: MSI-X ISR Rcv cmd -to- "
+ "cmd pass to NVME Layer\n");
+ len += snprintf(buf + len, PAGE_SIZE-len,
+ "avg:%08lld min:%08lld max %08lld\n",
+ div_u64(phba->ktime_seg1_total,
+ phba->ktime_data_samples),
+ phba->ktime_seg1_min,
+ phba->ktime_seg1_max);
+ len += snprintf(buf + len, PAGE_SIZE-len,
+ "Segment 2: cmd pass to NVME Layer- "
+ "-to- Driver rcv cmd OP (action)\n");
+ len += snprintf(buf + len, PAGE_SIZE-len,
+ "avg:%08lld min:%08lld max %08lld\n",
+ div_u64(phba->ktime_seg2_total,
+ phba->ktime_data_samples),
+ phba->ktime_seg2_min,
+ phba->ktime_seg2_max);
+ len += snprintf(buf + len, PAGE_SIZE-len,
+ "Segment 3: Driver rcv cmd OP -to- "
+ "Firmware WQ doorbell: cmd\n");
+ len += snprintf(buf + len, PAGE_SIZE-len,
+ "avg:%08lld min:%08lld max %08lld\n",
+ div_u64(phba->ktime_seg3_total,
+ phba->ktime_data_samples),
+ phba->ktime_seg3_min,
+ phba->ktime_seg3_max);
+ len += snprintf(buf + len, PAGE_SIZE-len,
+ "Segment 4: Firmware WQ doorbell: cmd "
+ "-to- MSI-X ISR for cmd cmpl\n");
+ len += snprintf(buf + len, PAGE_SIZE-len,
+ "avg:%08lld min:%08lld max %08lld\n",
+ div_u64(phba->ktime_seg4_total,
+ phba->ktime_data_samples),
+ phba->ktime_seg4_min,
+ phba->ktime_seg4_max);
+ len += snprintf(buf + len, PAGE_SIZE-len,
+ "Segment 5: MSI-X ISR for cmd cmpl "
+ "-to- NVME layer passed cmd done\n");
+ len += snprintf(buf + len, PAGE_SIZE-len,
+ "avg:%08lld min:%08lld max %08lld\n",
+ div_u64(phba->ktime_seg5_total,
+ phba->ktime_data_samples),
+ phba->ktime_seg5_min,
+ phba->ktime_seg5_max);
+
+ if (phba->ktime_status_samples == 0) {
+ len += snprintf(buf + len, PAGE_SIZE-len,
+ "Total: cmd received by MSI-X ISR "
+ "-to- cmd completed on wire\n");
+ len += snprintf(buf + len, PAGE_SIZE-len,
+ "avg:%08lld min:%08lld "
+ "max %08lld\n",
+ div_u64(phba->ktime_seg10_total,
+ phba->ktime_data_samples),
+ phba->ktime_seg10_min,
+ phba->ktime_seg10_max);
+ return len;
+ }
+
+ len += snprintf(buf + len, PAGE_SIZE-len,
+ "Segment 6: NVME layer passed cmd done "
+ "-to- Driver rcv rsp status OP\n");
+ len += snprintf(buf + len, PAGE_SIZE-len,
+ "avg:%08lld min:%08lld max %08lld\n",
+ div_u64(phba->ktime_seg6_total,
+ phba->ktime_status_samples),
+ phba->ktime_seg6_min,
+ phba->ktime_seg6_max);
+ len += snprintf(buf + len, PAGE_SIZE-len,
+ "Segment 7: Driver rcv rsp status OP "
+ "-to- Firmware WQ doorbell: status\n");
+ len += snprintf(buf + len, PAGE_SIZE-len,
+ "avg:%08lld min:%08lld max %08lld\n",
+ div_u64(phba->ktime_seg7_total,
+ phba->ktime_status_samples),
+ phba->ktime_seg7_min,
+ phba->ktime_seg7_max);
+ len += snprintf(buf + len, PAGE_SIZE-len,
+ "Segment 8: Firmware WQ doorbell: status"
+ " -to- MSI-X ISR for status cmpl\n");
+ len += snprintf(buf + len, PAGE_SIZE-len,
+ "avg:%08lld min:%08lld max %08lld\n",
+ div_u64(phba->ktime_seg8_total,
+ phba->ktime_status_samples),
+ phba->ktime_seg8_min,
+ phba->ktime_seg8_max);
+ len += snprintf(buf + len, PAGE_SIZE-len,
+ "Segment 9: MSI-X ISR for status cmpl "
+ "-to- NVME layer passed status done\n");
+ len += snprintf(buf + len, PAGE_SIZE-len,
+ "avg:%08lld min:%08lld max %08lld\n",
+ div_u64(phba->ktime_seg9_total,
+ phba->ktime_status_samples),
+ phba->ktime_seg9_min,
+ phba->ktime_seg9_max);
+ len += snprintf(buf + len, PAGE_SIZE-len,
+ "Total: cmd received by MSI-X ISR -to- "
+ "cmd completed on wire\n");
+ len += snprintf(buf + len, PAGE_SIZE-len,
+ "avg:%08lld min:%08lld max %08lld\n",
+ div_u64(phba->ktime_seg10_total,
+ phba->ktime_status_samples),
+ phba->ktime_seg10_min,
+ phba->ktime_seg10_max);
+ return len;
+}
+
+/**
+ * lpfc_debugfs_nvmeio_trc_data - Dump NVME IO trace list to a buffer
+ * @phba: The phba to gather target node info from.
+ * @buf: The buffer to dump log into.
+ * @size: The maximum amount of data to process.
+ *
+ * Description:
+ * This routine dumps the NVME IO trace associated with @phba
+ *
+ * Return Value:
+ * This routine returns the amount of bytes that were dumped into @buf and will
+ * not exceed @size.
+ **/
+static int
+lpfc_debugfs_nvmeio_trc_data(struct lpfc_hba *phba, char *buf, int size)
+{
+ struct lpfc_debugfs_nvmeio_trc *dtp;
+ int i, state, index, skip;
+ int len = 0;
+
+ state = phba->nvmeio_trc_on;
+
+ index = (atomic_read(&phba->nvmeio_trc_cnt) + 1) &
+ (phba->nvmeio_trc_size - 1);
+ skip = phba->nvmeio_trc_output_idx;
+
+ len += snprintf(buf + len, size - len,
+ "%s IO Trace %s: next_idx %d skip %d size %d\n",
+ (phba->nvmet_support ? "NVME" : "NVMET"),
+ (state ? "Enabled" : "Disabled"),
+ index, skip, phba->nvmeio_trc_size);
+
+ if (!phba->nvmeio_trc || state)
+ return len;
+
+ /* trace MUST bhe off to continue */
+
+ for (i = index; i < phba->nvmeio_trc_size; i++) {
+ if (skip) {
+ skip--;
+ continue;
+ }
+ dtp = phba->nvmeio_trc + i;
+ phba->nvmeio_trc_output_idx++;
+
+ if (!dtp->fmt)
+ continue;
+
+ len += snprintf(buf + len, size - len, dtp->fmt,
+ dtp->data1, dtp->data2, dtp->data3);
+
+ if (phba->nvmeio_trc_output_idx >= phba->nvmeio_trc_size) {
+ phba->nvmeio_trc_output_idx = 0;
+ len += snprintf(buf + len, size - len,
+ "Trace Complete\n");
+ goto out;
+ }
+
+ if (len >= (size - LPFC_DEBUG_OUT_LINE_SZ)) {
+ len += snprintf(buf + len, size - len,
+ "Trace Continue (%d of %d)\n",
+ phba->nvmeio_trc_output_idx,
+ phba->nvmeio_trc_size);
+ goto out;
+ }
+ }
+ for (i = 0; i < index; i++) {
+ if (skip) {
+ skip--;
+ continue;
+ }
+ dtp = phba->nvmeio_trc + i;
+ phba->nvmeio_trc_output_idx++;
+
+ if (!dtp->fmt)
+ continue;
+
+ len += snprintf(buf + len, size - len, dtp->fmt,
+ dtp->data1, dtp->data2, dtp->data3);
+
+ if (phba->nvmeio_trc_output_idx >= phba->nvmeio_trc_size) {
+ phba->nvmeio_trc_output_idx = 0;
+ len += snprintf(buf + len, size - len,
+ "Trace Complete\n");
+ goto out;
+ }
+
+ if (len >= (size - LPFC_DEBUG_OUT_LINE_SZ)) {
+ len += snprintf(buf + len, size - len,
+ "Trace Continue (%d of %d)\n",
+ phba->nvmeio_trc_output_idx,
+ phba->nvmeio_trc_size);
+ goto out;
+ }
+ }
+
+ len += snprintf(buf + len, size - len,
+ "Trace Done\n");
+out:
+ return len;
+}
+
+/**
+ * lpfc_debugfs_cpucheck_data - Dump target node list to a buffer
+ * @vport: The vport to gather target node info from.
+ * @buf: The buffer to dump log into.
+ * @size: The maximum amount of data to process.
+ *
+ * Description:
+ * This routine dumps the NVME statistics associated with @vport
+ *
+ * Return Value:
+ * This routine returns the amount of bytes that were dumped into @buf and will
+ * not exceed @size.
+ **/
+static int
+lpfc_debugfs_cpucheck_data(struct lpfc_vport *vport, char *buf, int size)
+{
+ struct lpfc_hba *phba = vport->phba;
+ int i;
+ int len = 0;
+ uint32_t tot_xmt = 0;
+ uint32_t tot_rcv = 0;
+ uint32_t tot_cmpl = 0;
+ uint32_t tot_ccmpl = 0;
+
+ if (phba->nvmet_support == 0) {
+ /* NVME Initiator */
+ len += snprintf(buf + len, PAGE_SIZE - len,
+ "CPUcheck %s\n",
+ (phba->cpucheck_on & LPFC_CHECK_NVME_IO ?
+ "Enabled" : "Disabled"));
+ for (i = 0; i < phba->sli4_hba.num_present_cpu; i++) {
+ if (i >= LPFC_CHECK_CPU_CNT)
+ break;
+ len += snprintf(buf + len, PAGE_SIZE - len,
+ "%02d: xmit x%08x cmpl x%08x\n",
+ i, phba->cpucheck_xmt_io[i],
+ phba->cpucheck_cmpl_io[i]);
+ tot_xmt += phba->cpucheck_xmt_io[i];
+ tot_cmpl += phba->cpucheck_cmpl_io[i];
+ }
+ len += snprintf(buf + len, PAGE_SIZE - len,
+ "tot:xmit x%08x cmpl x%08x\n",
+ tot_xmt, tot_cmpl);
+ return len;
+ }
+
+ /* NVME Target */
+ len += snprintf(buf + len, PAGE_SIZE - len,
+ "CPUcheck %s ",
+ (phba->cpucheck_on & LPFC_CHECK_NVMET_IO ?
+ "IO Enabled - " : "IO Disabled - "));
+ len += snprintf(buf + len, PAGE_SIZE - len,
+ "%s\n",
+ (phba->cpucheck_on & LPFC_CHECK_NVMET_RCV ?
+ "Rcv Enabled\n" : "Rcv Disabled\n"));
+ for (i = 0; i < phba->sli4_hba.num_present_cpu; i++) {
+ if (i >= LPFC_CHECK_CPU_CNT)
+ break;
+ len += snprintf(buf + len, PAGE_SIZE - len,
+ "%02d: xmit x%08x ccmpl x%08x "
+ "cmpl x%08x rcv x%08x\n",
+ i, phba->cpucheck_xmt_io[i],
+ phba->cpucheck_ccmpl_io[i],
+ phba->cpucheck_cmpl_io[i],
+ phba->cpucheck_rcv_io[i]);
+ tot_xmt += phba->cpucheck_xmt_io[i];
+ tot_rcv += phba->cpucheck_rcv_io[i];
+ tot_cmpl += phba->cpucheck_cmpl_io[i];
+ tot_ccmpl += phba->cpucheck_ccmpl_io[i];
+ }
+ len += snprintf(buf + len, PAGE_SIZE - len,
+ "tot:xmit x%08x ccmpl x%08x cmpl x%08x rcv x%08x\n",
+ tot_xmt, tot_ccmpl, tot_cmpl, tot_rcv);
return len;
}
+
#endif
/**
return;
}
+/**
+ * lpfc_debugfs_nvme_trc - Store NVME/NVMET trace log
+ * @phba: The phba to associate this trace string with for retrieval.
+ * @fmt: Format string to be displayed when dumping the log.
+ * @data1: 1st data parameter to be applied to @fmt.
+ * @data2: 2nd data parameter to be applied to @fmt.
+ * @data3: 3rd data parameter to be applied to @fmt.
+ *
+ * Description:
+ * This routine is used by the driver code to add a debugfs log entry to the
+ * nvme trace buffer associated with @phba. @fmt, @data1, @data2, and
+ * @data3 are used like printf when displaying the log.
+ **/
+inline void
+lpfc_debugfs_nvme_trc(struct lpfc_hba *phba, char *fmt,
+ uint16_t data1, uint16_t data2, uint32_t data3)
+{
+#ifdef CONFIG_SCSI_LPFC_DEBUG_FS
+ struct lpfc_debugfs_nvmeio_trc *dtp;
+ int index;
+
+ if (!phba->nvmeio_trc_on || !phba->nvmeio_trc)
+ return;
+
+ index = atomic_inc_return(&phba->nvmeio_trc_cnt) &
+ (phba->nvmeio_trc_size - 1);
+ dtp = phba->nvmeio_trc + index;
+ dtp->fmt = fmt;
+ dtp->data1 = data1;
+ dtp->data2 = data2;
+ dtp->data3 = data3;
+#endif
+}
+
#ifdef CONFIG_SCSI_LPFC_DEBUG_FS
/**
* lpfc_debugfs_disc_trc_open - Open the discovery trace log
goto out;
/* Round to page boundary */
- printk(KERN_ERR "9059 BLKGRD: %s: _dump_buf_data=0x%p\n",
+ pr_err("9059 BLKGRD: %s: _dump_buf_data=0x%p\n",
__func__, _dump_buf_data);
debug->buffer = _dump_buf_data;
if (!debug->buffer) {
goto out;
/* Round to page boundary */
- printk(KERN_ERR "9060 BLKGRD: %s: _dump_buf_dif=0x%p file=%pD\n",
- __func__, _dump_buf_dif, file);
+ pr_err("9060 BLKGRD: %s: _dump_buf_dif=0x%p file=%pD\n",
+ __func__, _dump_buf_dif, file);
debug->buffer = _dump_buf_dif;
if (!debug->buffer) {
kfree(debug);
return 0;
}
+
+static int
+lpfc_debugfs_nvmestat_open(struct inode *inode, struct file *file)
+{
+ struct lpfc_vport *vport = inode->i_private;
+ struct lpfc_debug *debug;
+ int rc = -ENOMEM;
+
+ debug = kmalloc(sizeof(*debug), GFP_KERNEL);
+ if (!debug)
+ goto out;
+
+ /* Round to page boundary */
+ debug->buffer = kmalloc(LPFC_NVMESTAT_SIZE, GFP_KERNEL);
+ if (!debug->buffer) {
+ kfree(debug);
+ goto out;
+ }
+
+ debug->len = lpfc_debugfs_nvmestat_data(vport, debug->buffer,
+ LPFC_NVMESTAT_SIZE);
+
+ debug->i_private = inode->i_private;
+ file->private_data = debug;
+
+ rc = 0;
+out:
+ return rc;
+}
+
+static ssize_t
+lpfc_debugfs_nvmestat_write(struct file *file, const char __user *buf,
+ size_t nbytes, loff_t *ppos)
+{
+ struct lpfc_debug *debug = file->private_data;
+ struct lpfc_vport *vport = (struct lpfc_vport *)debug->i_private;
+ struct lpfc_hba *phba = vport->phba;
+ struct lpfc_nvmet_tgtport *tgtp;
+ char mybuf[64];
+ char *pbuf;
+
+ if (!phba->targetport)
+ return -ENXIO;
+
+ if (nbytes > 64)
+ nbytes = 64;
+
+ /* Protect copy from user */
+ if (!access_ok(VERIFY_READ, buf, nbytes))
+ return -EFAULT;
+
+ memset(mybuf, 0, sizeof(mybuf));
+
+ if (copy_from_user(mybuf, buf, nbytes))
+ return -EFAULT;
+ pbuf = &mybuf[0];
+
+ tgtp = (struct lpfc_nvmet_tgtport *)phba->targetport->private;
+ if ((strncmp(pbuf, "reset", strlen("reset")) == 0) ||
+ (strncmp(pbuf, "zero", strlen("zero")) == 0)) {
+ atomic_set(&tgtp->rcv_ls_req_in, 0);
+ atomic_set(&tgtp->rcv_ls_req_out, 0);
+ atomic_set(&tgtp->rcv_ls_req_drop, 0);
+ atomic_set(&tgtp->xmt_ls_abort, 0);
+ atomic_set(&tgtp->xmt_ls_rsp, 0);
+ atomic_set(&tgtp->xmt_ls_drop, 0);
+ atomic_set(&tgtp->xmt_ls_rsp_error, 0);
+ atomic_set(&tgtp->xmt_ls_rsp_cmpl, 0);
+
+ atomic_set(&tgtp->rcv_fcp_cmd_in, 0);
+ atomic_set(&tgtp->rcv_fcp_cmd_out, 0);
+ atomic_set(&tgtp->rcv_fcp_cmd_drop, 0);
+ atomic_set(&tgtp->xmt_fcp_abort, 0);
+ atomic_set(&tgtp->xmt_fcp_drop, 0);
+ atomic_set(&tgtp->xmt_fcp_read_rsp, 0);
+ atomic_set(&tgtp->xmt_fcp_read, 0);
+ atomic_set(&tgtp->xmt_fcp_write, 0);
+ atomic_set(&tgtp->xmt_fcp_rsp, 0);
+ atomic_set(&tgtp->xmt_fcp_rsp_cmpl, 0);
+ atomic_set(&tgtp->xmt_fcp_rsp_error, 0);
+ atomic_set(&tgtp->xmt_fcp_rsp_drop, 0);
+
+ atomic_set(&tgtp->xmt_abort_rsp, 0);
+ atomic_set(&tgtp->xmt_abort_rsp_error, 0);
+ atomic_set(&tgtp->xmt_abort_cmpl, 0);
+ }
+ return nbytes;
+}
+
+static int
+lpfc_debugfs_nvmektime_open(struct inode *inode, struct file *file)
+{
+ struct lpfc_vport *vport = inode->i_private;
+ struct lpfc_debug *debug;
+ int rc = -ENOMEM;
+
+ debug = kmalloc(sizeof(*debug), GFP_KERNEL);
+ if (!debug)
+ goto out;
+
+ /* Round to page boundary */
+ debug->buffer = kmalloc(LPFC_NVMEKTIME_SIZE, GFP_KERNEL);
+ if (!debug->buffer) {
+ kfree(debug);
+ goto out;
+ }
+
+ debug->len = lpfc_debugfs_nvmektime_data(vport, debug->buffer,
+ LPFC_NVMEKTIME_SIZE);
+
+ debug->i_private = inode->i_private;
+ file->private_data = debug;
+
+ rc = 0;
+out:
+ return rc;
+}
+
+static ssize_t
+lpfc_debugfs_nvmektime_write(struct file *file, const char __user *buf,
+ size_t nbytes, loff_t *ppos)
+{
+ struct lpfc_debug *debug = file->private_data;
+ struct lpfc_vport *vport = (struct lpfc_vport *)debug->i_private;
+ struct lpfc_hba *phba = vport->phba;
+ char mybuf[64];
+ char *pbuf;
+
+ if (nbytes > 64)
+ nbytes = 64;
+
+ /* Protect copy from user */
+ if (!access_ok(VERIFY_READ, buf, nbytes))
+ return -EFAULT;
+
+ memset(mybuf, 0, sizeof(mybuf));
+
+ if (copy_from_user(mybuf, buf, nbytes))
+ return -EFAULT;
+ pbuf = &mybuf[0];
+
+ if ((strncmp(pbuf, "on", sizeof("on") - 1) == 0)) {
+ phba->ktime_data_samples = 0;
+ phba->ktime_status_samples = 0;
+ phba->ktime_seg1_total = 0;
+ phba->ktime_seg1_max = 0;
+ phba->ktime_seg1_min = 0xffffffff;
+ phba->ktime_seg2_total = 0;
+ phba->ktime_seg2_max = 0;
+ phba->ktime_seg2_min = 0xffffffff;
+ phba->ktime_seg3_total = 0;
+ phba->ktime_seg3_max = 0;
+ phba->ktime_seg3_min = 0xffffffff;
+ phba->ktime_seg4_total = 0;
+ phba->ktime_seg4_max = 0;
+ phba->ktime_seg4_min = 0xffffffff;
+ phba->ktime_seg5_total = 0;
+ phba->ktime_seg5_max = 0;
+ phba->ktime_seg5_min = 0xffffffff;
+ phba->ktime_seg6_total = 0;
+ phba->ktime_seg6_max = 0;
+ phba->ktime_seg6_min = 0xffffffff;
+ phba->ktime_seg7_total = 0;
+ phba->ktime_seg7_max = 0;
+ phba->ktime_seg7_min = 0xffffffff;
+ phba->ktime_seg8_total = 0;
+ phba->ktime_seg8_max = 0;
+ phba->ktime_seg8_min = 0xffffffff;
+ phba->ktime_seg9_total = 0;
+ phba->ktime_seg9_max = 0;
+ phba->ktime_seg9_min = 0xffffffff;
+ phba->ktime_seg10_total = 0;
+ phba->ktime_seg10_max = 0;
+ phba->ktime_seg10_min = 0xffffffff;
+
+ phba->ktime_on = 1;
+ return strlen(pbuf);
+ } else if ((strncmp(pbuf, "off",
+ sizeof("off") - 1) == 0)) {
+ phba->ktime_on = 0;
+ return strlen(pbuf);
+ } else if ((strncmp(pbuf, "zero",
+ sizeof("zero") - 1) == 0)) {
+ phba->ktime_data_samples = 0;
+ phba->ktime_status_samples = 0;
+ phba->ktime_seg1_total = 0;
+ phba->ktime_seg1_max = 0;
+ phba->ktime_seg1_min = 0xffffffff;
+ phba->ktime_seg2_total = 0;
+ phba->ktime_seg2_max = 0;
+ phba->ktime_seg2_min = 0xffffffff;
+ phba->ktime_seg3_total = 0;
+ phba->ktime_seg3_max = 0;
+ phba->ktime_seg3_min = 0xffffffff;
+ phba->ktime_seg4_total = 0;
+ phba->ktime_seg4_max = 0;
+ phba->ktime_seg4_min = 0xffffffff;
+ phba->ktime_seg5_total = 0;
+ phba->ktime_seg5_max = 0;
+ phba->ktime_seg5_min = 0xffffffff;
+ phba->ktime_seg6_total = 0;
+ phba->ktime_seg6_max = 0;
+ phba->ktime_seg6_min = 0xffffffff;
+ phba->ktime_seg7_total = 0;
+ phba->ktime_seg7_max = 0;
+ phba->ktime_seg7_min = 0xffffffff;
+ phba->ktime_seg8_total = 0;
+ phba->ktime_seg8_max = 0;
+ phba->ktime_seg8_min = 0xffffffff;
+ phba->ktime_seg9_total = 0;
+ phba->ktime_seg9_max = 0;
+ phba->ktime_seg9_min = 0xffffffff;
+ phba->ktime_seg10_total = 0;
+ phba->ktime_seg10_max = 0;
+ phba->ktime_seg10_min = 0xffffffff;
+ return strlen(pbuf);
+ }
+ return -EINVAL;
+}
+
+static int
+lpfc_debugfs_nvmeio_trc_open(struct inode *inode, struct file *file)
+{
+ struct lpfc_hba *phba = inode->i_private;
+ struct lpfc_debug *debug;
+ int rc = -ENOMEM;
+
+ debug = kmalloc(sizeof(*debug), GFP_KERNEL);
+ if (!debug)
+ goto out;
+
+ /* Round to page boundary */
+ debug->buffer = kmalloc(LPFC_NVMEIO_TRC_SIZE, GFP_KERNEL);
+ if (!debug->buffer) {
+ kfree(debug);
+ goto out;
+ }
+
+ debug->len = lpfc_debugfs_nvmeio_trc_data(phba, debug->buffer,
+ LPFC_NVMEIO_TRC_SIZE);
+
+ debug->i_private = inode->i_private;
+ file->private_data = debug;
+
+ rc = 0;
+out:
+ return rc;
+}
+
+static ssize_t
+lpfc_debugfs_nvmeio_trc_write(struct file *file, const char __user *buf,
+ size_t nbytes, loff_t *ppos)
+{
+ struct lpfc_debug *debug = file->private_data;
+ struct lpfc_hba *phba = (struct lpfc_hba *)debug->i_private;
+ int i;
+ unsigned long sz;
+ char mybuf[64];
+ char *pbuf;
+
+ if (nbytes > 64)
+ nbytes = 64;
+
+ /* Protect copy from user */
+ if (!access_ok(VERIFY_READ, buf, nbytes))
+ return -EFAULT;
+
+ memset(mybuf, 0, sizeof(mybuf));
+
+ if (copy_from_user(mybuf, buf, nbytes))
+ return -EFAULT;
+ pbuf = &mybuf[0];
+
+ if ((strncmp(pbuf, "off", sizeof("off") - 1) == 0)) {
+ lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+ "0570 nvmeio_trc_off\n");
+ phba->nvmeio_trc_output_idx = 0;
+ phba->nvmeio_trc_on = 0;
+ return strlen(pbuf);
+ } else if ((strncmp(pbuf, "on", sizeof("on") - 1) == 0)) {
+ lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+ "0571 nvmeio_trc_on\n");
+ phba->nvmeio_trc_output_idx = 0;
+ phba->nvmeio_trc_on = 1;
+ return strlen(pbuf);
+ }
+
+ /* We must be off to allocate the trace buffer */
+ if (phba->nvmeio_trc_on != 0)
+ return -EINVAL;
+
+ /* If not on or off, the parameter is the trace buffer size */
+ i = kstrtoul(pbuf, 0, &sz);
+ if (i)
+ return -EINVAL;
+ phba->nvmeio_trc_size = (uint32_t)sz;
+
+ /* It must be a power of 2 - round down */
+ i = 0;
+ while (sz > 1) {
+ sz = sz >> 1;
+ i++;
+ }
+ sz = (1 << i);
+ if (phba->nvmeio_trc_size != sz)
+ lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+ "0572 nvmeio_trc_size changed to %ld\n",
+ sz);
+ phba->nvmeio_trc_size = (uint32_t)sz;
+
+ /* If one previously exists, free it */
+ kfree(phba->nvmeio_trc);
+
+ /* Allocate new trace buffer and initialize */
+ phba->nvmeio_trc = kmalloc((sizeof(struct lpfc_debugfs_nvmeio_trc) *
+ sz), GFP_KERNEL);
+ if (!phba->nvmeio_trc) {
+ lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+ "0573 Cannot create debugfs "
+ "nvmeio_trc buffer\n");
+ return -ENOMEM;
+ }
+ memset(phba->nvmeio_trc, 0,
+ (sizeof(struct lpfc_debugfs_nvmeio_trc) * sz));
+ atomic_set(&phba->nvmeio_trc_cnt, 0);
+ phba->nvmeio_trc_on = 0;
+ phba->nvmeio_trc_output_idx = 0;
+
+ return strlen(pbuf);
+}
+
+static int
+lpfc_debugfs_cpucheck_open(struct inode *inode, struct file *file)
+{
+ struct lpfc_vport *vport = inode->i_private;
+ struct lpfc_debug *debug;
+ int rc = -ENOMEM;
+
+ debug = kmalloc(sizeof(*debug), GFP_KERNEL);
+ if (!debug)
+ goto out;
+
+ /* Round to page boundary */
+ debug->buffer = kmalloc(LPFC_CPUCHECK_SIZE, GFP_KERNEL);
+ if (!debug->buffer) {
+ kfree(debug);
+ goto out;
+ }
+
+ debug->len = lpfc_debugfs_cpucheck_data(vport, debug->buffer,
+ LPFC_NVMEKTIME_SIZE);
+
+ debug->i_private = inode->i_private;
+ file->private_data = debug;
+
+ rc = 0;
+out:
+ return rc;
+}
+
+static ssize_t
+lpfc_debugfs_cpucheck_write(struct file *file, const char __user *buf,
+ size_t nbytes, loff_t *ppos)
+{
+ struct lpfc_debug *debug = file->private_data;
+ struct lpfc_vport *vport = (struct lpfc_vport *)debug->i_private;
+ struct lpfc_hba *phba = vport->phba;
+ char mybuf[64];
+ char *pbuf;
+ int i;
+
+ if (nbytes > 64)
+ nbytes = 64;
+
+ /* Protect copy from user */
+ if (!access_ok(VERIFY_READ, buf, nbytes))
+ return -EFAULT;
+
+ memset(mybuf, 0, sizeof(mybuf));
+
+ if (copy_from_user(mybuf, buf, nbytes))
+ return -EFAULT;
+ pbuf = &mybuf[0];
+
+ if ((strncmp(pbuf, "on", sizeof("on") - 1) == 0)) {
+ if (phba->nvmet_support)
+ phba->cpucheck_on |= LPFC_CHECK_NVMET_IO;
+ else
+ phba->cpucheck_on |= LPFC_CHECK_NVME_IO;
+ return strlen(pbuf);
+ } else if ((strncmp(pbuf, "rcv",
+ sizeof("rcv") - 1) == 0)) {
+ if (phba->nvmet_support)
+ phba->cpucheck_on |= LPFC_CHECK_NVMET_RCV;
+ else
+ return -EINVAL;
+ return strlen(pbuf);
+ } else if ((strncmp(pbuf, "off",
+ sizeof("off") - 1) == 0)) {
+ phba->cpucheck_on = LPFC_CHECK_OFF;
+ return strlen(pbuf);
+ } else if ((strncmp(pbuf, "zero",
+ sizeof("zero") - 1) == 0)) {
+ for (i = 0; i < phba->sli4_hba.num_present_cpu; i++) {
+ if (i >= LPFC_CHECK_CPU_CNT)
+ break;
+ phba->cpucheck_rcv_io[i] = 0;
+ phba->cpucheck_xmt_io[i] = 0;
+ phba->cpucheck_cmpl_io[i] = 0;
+ phba->cpucheck_ccmpl_io[i] = 0;
+ }
+ return strlen(pbuf);
+ }
+ return -EINVAL;
+}
+
/*
* ---------------------------------
* iDiag debugfs file access methods
return -EINVAL;
}
+static int
+__lpfc_idiag_print_wq(struct lpfc_queue *qp, char *wqtype,
+ char *pbuffer, int len)
+{
+ if (!qp)
+ return len;
+
+ len += snprintf(pbuffer + len, LPFC_QUE_INFO_GET_BUF_SIZE - len,
+ "\t\t%s WQ info: ", wqtype);
+ len += snprintf(pbuffer + len, LPFC_QUE_INFO_GET_BUF_SIZE - len,
+ "AssocCQID[%04d]: WQ-STAT[oflow:x%x posted:x%llx]\n",
+ qp->assoc_qid, qp->q_cnt_1,
+ (unsigned long long)qp->q_cnt_4);
+ len += snprintf(pbuffer + len, LPFC_QUE_INFO_GET_BUF_SIZE - len,
+ "\t\tWQID[%02d], QE-CNT[%04d], QE-SIZE[%04d], "
+ "HOST-IDX[%04d], PORT-IDX[%04d]",
+ qp->queue_id, qp->entry_count,
+ qp->entry_size, qp->host_index,
+ qp->hba_index);
+ len += snprintf(pbuffer + len,
+ LPFC_QUE_INFO_GET_BUF_SIZE - len, "\n");
+ return len;
+}
+
+static int
+lpfc_idiag_wqs_for_cq(struct lpfc_hba *phba, char *wqtype, char *pbuffer,
+ int *len, int max_cnt, int cq_id)
+{
+ struct lpfc_queue *qp;
+ int qidx;
+
+ for (qidx = 0; qidx < phba->cfg_fcp_io_channel; qidx++) {
+ qp = phba->sli4_hba.fcp_wq[qidx];
+ if (qp->assoc_qid != cq_id)
+ continue;
+ *len = __lpfc_idiag_print_wq(qp, wqtype, pbuffer, *len);
+ if (*len >= max_cnt)
+ return 1;
+ }
+ for (qidx = 0; qidx < phba->cfg_nvme_io_channel; qidx++) {
+ qp = phba->sli4_hba.nvme_wq[qidx];
+ if (qp->assoc_qid != cq_id)
+ continue;
+ *len = __lpfc_idiag_print_wq(qp, wqtype, pbuffer, *len);
+ if (*len >= max_cnt)
+ return 1;
+ }
+ return 0;
+}
+
+static int
+__lpfc_idiag_print_cq(struct lpfc_queue *qp, char *cqtype,
+ char *pbuffer, int len)
+{
+ if (!qp)
+ return len;
+
+ len += snprintf(pbuffer + len, LPFC_QUE_INFO_GET_BUF_SIZE - len,
+ "\t%s CQ info: ", cqtype);
+ len += snprintf(pbuffer + len, LPFC_QUE_INFO_GET_BUF_SIZE - len,
+ "AssocEQID[%02d]: CQ STAT[max:x%x relw:x%x "
+ "xabt:x%x wq:x%llx]\n",
+ qp->assoc_qid, qp->q_cnt_1, qp->q_cnt_2,
+ qp->q_cnt_3, (unsigned long long)qp->q_cnt_4);
+ len += snprintf(pbuffer + len, LPFC_QUE_INFO_GET_BUF_SIZE - len,
+ "\tCQID[%02d], QE-CNT[%04d], QE-SIZE[%04d], "
+ "HOST-IDX[%04d], PORT-IDX[%04d]",
+ qp->queue_id, qp->entry_count,
+ qp->entry_size, qp->host_index,
+ qp->hba_index);
+
+ len += snprintf(pbuffer + len, LPFC_QUE_INFO_GET_BUF_SIZE - len, "\n");
+
+ return len;
+}
+
+static int
+__lpfc_idiag_print_rqpair(struct lpfc_queue *qp, struct lpfc_queue *datqp,
+ char *rqtype, char *pbuffer, int len)
+{
+ if (!qp || !datqp)
+ return len;
+
+ len += snprintf(pbuffer + len, LPFC_QUE_INFO_GET_BUF_SIZE - len,
+ "\t\t%s RQ info: ", rqtype);
+ len += snprintf(pbuffer + len, LPFC_QUE_INFO_GET_BUF_SIZE - len,
+ "AssocCQID[%02d]: RQ-STAT[nopost:x%x nobuf:x%x "
+ "trunc:x%x rcv:x%llx]\n",
+ qp->assoc_qid, qp->q_cnt_1, qp->q_cnt_2,
+ qp->q_cnt_3, (unsigned long long)qp->q_cnt_4);
+ len += snprintf(pbuffer + len, LPFC_QUE_INFO_GET_BUF_SIZE - len,
+ "\t\tHQID[%02d], QE-CNT[%04d], QE-SIZE[%04d], "
+ "HOST-IDX[%04d], PORT-IDX[%04d]\n",
+ qp->queue_id, qp->entry_count, qp->entry_size,
+ qp->host_index, qp->hba_index);
+ len += snprintf(pbuffer + len, LPFC_QUE_INFO_GET_BUF_SIZE - len,
+ "\t\tDQID[%02d], QE-CNT[%04d], QE-SIZE[%04d], "
+ "HOST-IDX[%04d], PORT-IDX[%04d]\n",
+ datqp->queue_id, datqp->entry_count,
+ datqp->entry_size, datqp->host_index,
+ datqp->hba_index);
+ len += snprintf(pbuffer + len, LPFC_QUE_INFO_GET_BUF_SIZE - len, "\n");
+
+ return len;
+}
+
+static int
+lpfc_idiag_cqs_for_eq(struct lpfc_hba *phba, char *pbuffer,
+ int *len, int max_cnt, int eqidx, int eq_id)
+{
+ struct lpfc_queue *qp;
+ int qidx, rc;
+
+ for (qidx = 0; qidx < phba->cfg_fcp_io_channel; qidx++) {
+ qp = phba->sli4_hba.fcp_cq[qidx];
+ if (qp->assoc_qid != eq_id)
+ continue;
+
+ *len = __lpfc_idiag_print_cq(qp, "FCP", pbuffer, *len);
+
+ /* Reset max counter */
+ qp->CQ_max_cqe = 0;
+
+ if (*len >= max_cnt)
+ return 1;
+
+ rc = lpfc_idiag_wqs_for_cq(phba, "FCP", pbuffer, len,
+ max_cnt, qp->queue_id);
+ if (rc)
+ return 1;
+ }
+
+ for (qidx = 0; qidx < phba->cfg_nvme_io_channel; qidx++) {
+ qp = phba->sli4_hba.nvme_cq[qidx];
+ if (qp->assoc_qid != eq_id)
+ continue;
+
+ *len = __lpfc_idiag_print_cq(qp, "NVME", pbuffer, *len);
+
+ /* Reset max counter */
+ qp->CQ_max_cqe = 0;
+
+ if (*len >= max_cnt)
+ return 1;
+
+ rc = lpfc_idiag_wqs_for_cq(phba, "NVME", pbuffer, len,
+ max_cnt, qp->queue_id);
+ if (rc)
+ return 1;
+ }
+
+ if (eqidx < phba->cfg_nvmet_mrq) {
+ /* NVMET CQset */
+ qp = phba->sli4_hba.nvmet_cqset[eqidx];
+ *len = __lpfc_idiag_print_cq(qp, "NVMET CQset", pbuffer, *len);
+
+ /* Reset max counter */
+ qp->CQ_max_cqe = 0;
+
+ if (*len >= max_cnt)
+ return 1;
+
+ /* RQ header */
+ qp = phba->sli4_hba.nvmet_mrq_hdr[eqidx];
+ *len = __lpfc_idiag_print_rqpair(qp,
+ phba->sli4_hba.nvmet_mrq_data[eqidx],
+ "NVMET MRQ", pbuffer, *len);
+
+ if (*len >= max_cnt)
+ return 1;
+ }
+
+ return 0;
+}
+
+static int
+__lpfc_idiag_print_eq(struct lpfc_queue *qp, char *eqtype,
+ char *pbuffer, int len)
+{
+ if (!qp)
+ return len;
+
+ len += snprintf(pbuffer + len, LPFC_QUE_INFO_GET_BUF_SIZE - len,
+ "\n%s EQ info: EQ-STAT[max:x%x noE:x%x "
+ "bs:x%x proc:x%llx]\n",
+ eqtype, qp->q_cnt_1, qp->q_cnt_2, qp->q_cnt_3,
+ (unsigned long long)qp->q_cnt_4);
+ len += snprintf(pbuffer + len, LPFC_QUE_INFO_GET_BUF_SIZE - len,
+ "EQID[%02d], QE-CNT[%04d], QE-SIZE[%04d], "
+ "HOST-IDX[%04d], PORT-IDX[%04d]",
+ qp->queue_id, qp->entry_count, qp->entry_size,
+ qp->host_index, qp->hba_index);
+ len += snprintf(pbuffer + len, LPFC_QUE_INFO_GET_BUF_SIZE - len, "\n");
+
+ return len;
+}
+
/**
* lpfc_idiag_queinfo_read - idiag debugfs read queue information
* @file: The file pointer to read from.
* Description:
* This routine reads data from the @phba SLI4 PCI function queue information,
* and copies to user @buf.
+ * This routine only returns 1 EQs worth of information. It remembers the last
+ * EQ read and jumps to the next EQ. Thus subsequent calls to queInfo will
+ * retrieve all EQs allocated for the phba.
*
* Returns:
* This function returns the amount of data that was read (this could be less
{
struct lpfc_debug *debug = file->private_data;
struct lpfc_hba *phba = (struct lpfc_hba *)debug->i_private;
- int len = 0;
char *pbuffer;
- int x, cnt;
- int max_cnt;
+ int max_cnt, rc, x, len = 0;
struct lpfc_queue *qp = NULL;
-
if (!debug->buffer)
debug->buffer = kmalloc(LPFC_QUE_INFO_GET_BUF_SIZE, GFP_KERNEL);
if (!debug->buffer)
return 0;
pbuffer = debug->buffer;
- max_cnt = LPFC_QUE_INFO_GET_BUF_SIZE - 128;
+ max_cnt = LPFC_QUE_INFO_GET_BUF_SIZE - 256;
if (*ppos)
return 0;
spin_lock_irq(&phba->hbalock);
/* Fast-path event queue */
- if (phba->sli4_hba.hba_eq && phba->cfg_fcp_io_channel) {
- cnt = phba->cfg_fcp_io_channel;
+ if (phba->sli4_hba.hba_eq && phba->io_channel_irqs) {
+
+ x = phba->lpfc_idiag_last_eq;
+ if (phba->cfg_fof && (x >= phba->io_channel_irqs)) {
+ phba->lpfc_idiag_last_eq = 0;
+ goto fof;
+ }
+ phba->lpfc_idiag_last_eq++;
+ if (phba->lpfc_idiag_last_eq >= phba->io_channel_irqs)
+ if (phba->cfg_fof == 0)
+ phba->lpfc_idiag_last_eq = 0;
- for (x = 0; x < cnt; x++) {
+ len += snprintf(pbuffer + len, LPFC_QUE_INFO_GET_BUF_SIZE - len,
+ "EQ %d out of %d HBA EQs\n",
+ x, phba->io_channel_irqs);
- /* Fast-path EQ */
- qp = phba->sli4_hba.hba_eq[x];
- if (!qp)
- goto proc_cq;
+ /* Fast-path EQ */
+ qp = phba->sli4_hba.hba_eq[x];
+ if (!qp)
+ goto out;
- len += snprintf(pbuffer+len,
- LPFC_QUE_INFO_GET_BUF_SIZE-len,
- "\nHBA EQ info: "
- "EQ-STAT[max:x%x noE:x%x "
- "bs:x%x proc:x%llx]\n",
- qp->q_cnt_1, qp->q_cnt_2,
- qp->q_cnt_3, (unsigned long long)qp->q_cnt_4);
+ len = __lpfc_idiag_print_eq(qp, "HBA", pbuffer, len);
- len += snprintf(pbuffer+len,
- LPFC_QUE_INFO_GET_BUF_SIZE-len,
- "EQID[%02d], "
- "QE-CNT[%04d], QE-SIZE[%04d], "
- "HOST-IDX[%04d], PORT-IDX[%04d]",
- qp->queue_id,
- qp->entry_count,
- qp->entry_size,
- qp->host_index,
- qp->hba_index);
-
-
- /* Reset max counter */
- qp->EQ_max_eqe = 0;
+ /* Reset max counter */
+ qp->EQ_max_eqe = 0;
- len += snprintf(pbuffer+len,
- LPFC_QUE_INFO_GET_BUF_SIZE-len, "\n");
- if (len >= max_cnt)
- goto too_big;
-proc_cq:
- /* Fast-path FCP CQ */
- qp = phba->sli4_hba.fcp_cq[x];
- len += snprintf(pbuffer+len,
- LPFC_QUE_INFO_GET_BUF_SIZE-len,
- "\tFCP CQ info: ");
- len += snprintf(pbuffer+len,
- LPFC_QUE_INFO_GET_BUF_SIZE-len,
- "AssocEQID[%02d]: "
- "CQ STAT[max:x%x relw:x%x "
- "xabt:x%x wq:x%llx]\n",
- qp->assoc_qid,
- qp->q_cnt_1, qp->q_cnt_2,
- qp->q_cnt_3, (unsigned long long)qp->q_cnt_4);
- len += snprintf(pbuffer+len,
- LPFC_QUE_INFO_GET_BUF_SIZE-len,
- "\tCQID[%02d], "
- "QE-CNT[%04d], QE-SIZE[%04d], "
- "HOST-IDX[%04d], PORT-IDX[%04d]",
- qp->queue_id, qp->entry_count,
- qp->entry_size, qp->host_index,
- qp->hba_index);
+ if (len >= max_cnt)
+ goto too_big;
+ /* will dump both fcp and nvme cqs/wqs for the eq */
+ rc = lpfc_idiag_cqs_for_eq(phba, pbuffer, &len,
+ max_cnt, x, qp->queue_id);
+ if (rc)
+ goto too_big;
- /* Reset max counter */
- qp->CQ_max_cqe = 0;
+ /* Only EQ 0 has slow path CQs configured */
+ if (x)
+ goto out;
- len += snprintf(pbuffer+len,
- LPFC_QUE_INFO_GET_BUF_SIZE-len, "\n");
- if (len >= max_cnt)
- goto too_big;
+ /* Slow-path mailbox CQ */
+ qp = phba->sli4_hba.mbx_cq;
+ len = __lpfc_idiag_print_cq(qp, "MBX", pbuffer, len);
+ if (len >= max_cnt)
+ goto too_big;
- /* Fast-path FCP WQ */
- qp = phba->sli4_hba.fcp_wq[x];
+ /* Slow-path MBOX MQ */
+ qp = phba->sli4_hba.mbx_wq;
+ len = __lpfc_idiag_print_wq(qp, "MBX", pbuffer, len);
+ if (len >= max_cnt)
+ goto too_big;
- len += snprintf(pbuffer+len,
- LPFC_QUE_INFO_GET_BUF_SIZE-len,
- "\t\tFCP WQ info: ");
- len += snprintf(pbuffer+len,
- LPFC_QUE_INFO_GET_BUF_SIZE-len,
- "AssocCQID[%02d]: "
- "WQ-STAT[oflow:x%x posted:x%llx]\n",
- qp->assoc_qid,
- qp->q_cnt_1, (unsigned long long)qp->q_cnt_4);
- len += snprintf(pbuffer+len,
- LPFC_QUE_INFO_GET_BUF_SIZE-len,
- "\t\tWQID[%02d], "
- "QE-CNT[%04d], QE-SIZE[%04d], "
- "HOST-IDX[%04d], PORT-IDX[%04d]",
- qp->queue_id,
- qp->entry_count,
- qp->entry_size,
- qp->host_index,
- qp->hba_index);
-
- len += snprintf(pbuffer+len,
- LPFC_QUE_INFO_GET_BUF_SIZE-len, "\n");
- if (len >= max_cnt)
- goto too_big;
-
- if (x)
- continue;
-
- /* Only EQ 0 has slow path CQs configured */
-
- /* Slow-path mailbox CQ */
- qp = phba->sli4_hba.mbx_cq;
- if (qp) {
- len += snprintf(pbuffer+len,
- LPFC_QUE_INFO_GET_BUF_SIZE-len,
- "\tMBX CQ info: ");
- len += snprintf(pbuffer+len,
- LPFC_QUE_INFO_GET_BUF_SIZE-len,
- "AssocEQID[%02d]: "
- "CQ-STAT[mbox:x%x relw:x%x "
- "xabt:x%x wq:x%llx]\n",
- qp->assoc_qid,
- qp->q_cnt_1, qp->q_cnt_2,
- qp->q_cnt_3,
- (unsigned long long)qp->q_cnt_4);
- len += snprintf(pbuffer+len,
- LPFC_QUE_INFO_GET_BUF_SIZE-len,
- "\tCQID[%02d], "
- "QE-CNT[%04d], QE-SIZE[%04d], "
- "HOST-IDX[%04d], PORT-IDX[%04d]",
- qp->queue_id, qp->entry_count,
- qp->entry_size, qp->host_index,
- qp->hba_index);
-
- len += snprintf(pbuffer+len,
- LPFC_QUE_INFO_GET_BUF_SIZE-len, "\n");
- if (len >= max_cnt)
- goto too_big;
- }
+ /* Slow-path ELS response CQ */
+ qp = phba->sli4_hba.els_cq;
+ len = __lpfc_idiag_print_cq(qp, "ELS", pbuffer, len);
+ /* Reset max counter */
+ if (qp)
+ qp->CQ_max_cqe = 0;
+ if (len >= max_cnt)
+ goto too_big;
- /* Slow-path MBOX MQ */
- qp = phba->sli4_hba.mbx_wq;
- if (qp) {
- len += snprintf(pbuffer+len,
- LPFC_QUE_INFO_GET_BUF_SIZE-len,
- "\t\tMBX MQ info: ");
- len += snprintf(pbuffer+len,
- LPFC_QUE_INFO_GET_BUF_SIZE-len,
- "AssocCQID[%02d]:\n",
- phba->sli4_hba.mbx_wq->assoc_qid);
- len += snprintf(pbuffer+len,
- LPFC_QUE_INFO_GET_BUF_SIZE-len,
- "\t\tWQID[%02d], "
- "QE-CNT[%04d], QE-SIZE[%04d], "
- "HOST-IDX[%04d], PORT-IDX[%04d]",
- qp->queue_id, qp->entry_count,
- qp->entry_size, qp->host_index,
- qp->hba_index);
-
- len += snprintf(pbuffer+len,
- LPFC_QUE_INFO_GET_BUF_SIZE-len, "\n");
- if (len >= max_cnt)
- goto too_big;
- }
+ /* Slow-path ELS WQ */
+ qp = phba->sli4_hba.els_wq;
+ len = __lpfc_idiag_print_wq(qp, "ELS", pbuffer, len);
+ if (len >= max_cnt)
+ goto too_big;
- /* Slow-path ELS response CQ */
- qp = phba->sli4_hba.els_cq;
- if (qp) {
- len += snprintf(pbuffer+len,
- LPFC_QUE_INFO_GET_BUF_SIZE-len,
- "\tELS CQ info: ");
- len += snprintf(pbuffer+len,
- LPFC_QUE_INFO_GET_BUF_SIZE-len,
- "AssocEQID[%02d]: "
- "CQ-STAT[max:x%x relw:x%x "
- "xabt:x%x wq:x%llx]\n",
- qp->assoc_qid,
- qp->q_cnt_1, qp->q_cnt_2,
- qp->q_cnt_3,
- (unsigned long long)qp->q_cnt_4);
- len += snprintf(pbuffer+len,
- LPFC_QUE_INFO_GET_BUF_SIZE-len,
- "\tCQID [%02d], "
- "QE-CNT[%04d], QE-SIZE[%04d], "
- "HOST-IDX[%04d], PORT-IDX[%04d]",
- qp->queue_id, qp->entry_count,
- qp->entry_size, qp->host_index,
- qp->hba_index);
-
- /* Reset max counter */
- qp->CQ_max_cqe = 0;
-
- len += snprintf(pbuffer+len,
- LPFC_QUE_INFO_GET_BUF_SIZE-len, "\n");
- if (len >= max_cnt)
- goto too_big;
- }
+ /* Slow-path NVME LS response CQ */
+ qp = phba->sli4_hba.nvmels_cq;
+ len = __lpfc_idiag_print_cq(qp, "NVME LS",
+ pbuffer, len);
+ /* Reset max counter */
+ if (qp)
+ qp->CQ_max_cqe = 0;
+ if (len >= max_cnt)
+ goto too_big;
- /* Slow-path ELS WQ */
- qp = phba->sli4_hba.els_wq;
- if (qp) {
- len += snprintf(pbuffer+len,
- LPFC_QUE_INFO_GET_BUF_SIZE-len,
- "\t\tELS WQ info: ");
- len += snprintf(pbuffer+len,
- LPFC_QUE_INFO_GET_BUF_SIZE-len,
- "AssocCQID[%02d]: "
- " WQ-STAT[oflow:x%x "
- "posted:x%llx]\n",
- qp->assoc_qid,
- qp->q_cnt_1,
- (unsigned long long)qp->q_cnt_4);
- len += snprintf(pbuffer+len,
- LPFC_QUE_INFO_GET_BUF_SIZE-len,
- "\t\tWQID[%02d], "
- "QE-CNT[%04d], QE-SIZE[%04d], "
- "HOST-IDX[%04d], PORT-IDX[%04d]",
- qp->queue_id, qp->entry_count,
- qp->entry_size, qp->host_index,
- qp->hba_index);
-
- len += snprintf(pbuffer+len,
- LPFC_QUE_INFO_GET_BUF_SIZE-len, "\n");
- if (len >= max_cnt)
- goto too_big;
- }
+ /* Slow-path NVME LS WQ */
+ qp = phba->sli4_hba.nvmels_wq;
+ len = __lpfc_idiag_print_wq(qp, "NVME LS",
+ pbuffer, len);
+ if (len >= max_cnt)
+ goto too_big;
- if (phba->sli4_hba.hdr_rq && phba->sli4_hba.dat_rq) {
- /* Slow-path RQ header */
- qp = phba->sli4_hba.hdr_rq;
+ qp = phba->sli4_hba.hdr_rq;
+ len = __lpfc_idiag_print_rqpair(qp, phba->sli4_hba.dat_rq,
+ "RQpair", pbuffer, len);
+ if (len >= max_cnt)
+ goto too_big;
- len += snprintf(pbuffer+len,
- LPFC_QUE_INFO_GET_BUF_SIZE-len,
- "\t\tRQ info: ");
- len += snprintf(pbuffer+len,
- LPFC_QUE_INFO_GET_BUF_SIZE-len,
- "AssocCQID[%02d]: "
- "RQ-STAT[nopost:x%x nobuf:x%x "
- "trunc:x%x rcv:x%llx]\n",
- qp->assoc_qid,
- qp->q_cnt_1, qp->q_cnt_2,
- qp->q_cnt_3,
- (unsigned long long)qp->q_cnt_4);
- len += snprintf(pbuffer+len,
- LPFC_QUE_INFO_GET_BUF_SIZE-len,
- "\t\tHQID[%02d], "
- "QE-CNT[%04d], QE-SIZE[%04d], "
- "HOST-IDX[%04d], PORT-IDX[%04d]\n",
- qp->queue_id,
- qp->entry_count,
- qp->entry_size,
- qp->host_index,
- qp->hba_index);
-
- /* Slow-path RQ data */
- qp = phba->sli4_hba.dat_rq;
- len += snprintf(pbuffer+len,
- LPFC_QUE_INFO_GET_BUF_SIZE-len,
- "\t\tDQID[%02d], "
- "QE-CNT[%04d], QE-SIZE[%04d], "
- "HOST-IDX[%04d], PORT-IDX[%04d]\n",
- qp->queue_id,
- qp->entry_count,
- qp->entry_size,
- qp->host_index,
- qp->hba_index);
-
- len += snprintf(pbuffer+len,
- LPFC_QUE_INFO_GET_BUF_SIZE-len, "\n");
- }
- }
+ goto out;
}
+fof:
if (phba->cfg_fof) {
/* FOF EQ */
qp = phba->sli4_hba.fof_eq;
- if (!qp)
- goto out;
-
- len += snprintf(pbuffer+len,
- LPFC_QUE_INFO_GET_BUF_SIZE-len,
- "\nFOF EQ info: "
- "EQ-STAT[max:x%x noE:x%x "
- "bs:x%x proc:x%llx]\n",
- qp->q_cnt_1, qp->q_cnt_2,
- qp->q_cnt_3, (unsigned long long)qp->q_cnt_4);
-
- len += snprintf(pbuffer+len,
- LPFC_QUE_INFO_GET_BUF_SIZE-len,
- "EQID[%02d], "
- "QE-CNT[%04d], QE-SIZE[%04d], "
- "HOST-IDX[%04d], PORT-IDX[%04d]",
- qp->queue_id,
- qp->entry_count,
- qp->entry_size,
- qp->host_index,
- qp->hba_index);
+ len = __lpfc_idiag_print_eq(qp, "FOF", pbuffer, len);
/* Reset max counter */
- qp->EQ_max_eqe = 0;
+ if (qp)
+ qp->EQ_max_eqe = 0;
- len += snprintf(pbuffer+len,
- LPFC_QUE_INFO_GET_BUF_SIZE-len, "\n");
if (len >= max_cnt)
goto too_big;
- }
-
- if (phba->cfg_fof) {
/* OAS CQ */
qp = phba->sli4_hba.oas_cq;
- if (qp) {
- len += snprintf(pbuffer+len,
- LPFC_QUE_INFO_GET_BUF_SIZE-len,
- "\tOAS CQ info: ");
- len += snprintf(pbuffer+len,
- LPFC_QUE_INFO_GET_BUF_SIZE-len,
- "AssocEQID[%02d]: "
- "CQ STAT[max:x%x relw:x%x "
- "xabt:x%x wq:x%llx]\n",
- qp->assoc_qid,
- qp->q_cnt_1, qp->q_cnt_2,
- qp->q_cnt_3, (unsigned long long)qp->q_cnt_4);
- len += snprintf(pbuffer+len,
- LPFC_QUE_INFO_GET_BUF_SIZE-len,
- "\tCQID[%02d], "
- "QE-CNT[%04d], QE-SIZE[%04d], "
- "HOST-IDX[%04d], PORT-IDX[%04d]",
- qp->queue_id, qp->entry_count,
- qp->entry_size, qp->host_index,
- qp->hba_index);
-
- /* Reset max counter */
+ len = __lpfc_idiag_print_cq(qp, "OAS", pbuffer, len);
+ /* Reset max counter */
+ if (qp)
qp->CQ_max_cqe = 0;
-
- len += snprintf(pbuffer+len,
- LPFC_QUE_INFO_GET_BUF_SIZE-len, "\n");
- if (len >= max_cnt)
- goto too_big;
- }
+ if (len >= max_cnt)
+ goto too_big;
/* OAS WQ */
qp = phba->sli4_hba.oas_wq;
- if (qp) {
- len += snprintf(pbuffer+len,
- LPFC_QUE_INFO_GET_BUF_SIZE-len,
- "\t\tOAS WQ info: ");
- len += snprintf(pbuffer+len,
- LPFC_QUE_INFO_GET_BUF_SIZE-len,
- "AssocCQID[%02d]: "
- "WQ-STAT[oflow:x%x posted:x%llx]\n",
- qp->assoc_qid,
- qp->q_cnt_1, (unsigned long long)qp->q_cnt_4);
- len += snprintf(pbuffer+len,
- LPFC_QUE_INFO_GET_BUF_SIZE-len,
- "\t\tWQID[%02d], "
- "QE-CNT[%04d], QE-SIZE[%04d], "
- "HOST-IDX[%04d], PORT-IDX[%04d]",
- qp->queue_id,
- qp->entry_count,
- qp->entry_size,
- qp->host_index,
- qp->hba_index);
-
- len += snprintf(pbuffer+len,
- LPFC_QUE_INFO_GET_BUF_SIZE-len, "\n");
- if (len >= max_cnt)
- goto too_big;
- }
+ len = __lpfc_idiag_print_wq(qp, "OAS", pbuffer, len);
+ if (len >= max_cnt)
+ goto too_big;
}
-out:
+
spin_unlock_irq(&phba->hbalock);
return simple_read_from_buffer(buf, nbytes, ppos, pbuffer, len);
too_big:
- len += snprintf(pbuffer+len,
- LPFC_QUE_INFO_GET_BUF_SIZE-len, "Truncated ...\n");
+ len += snprintf(pbuffer + len,
+ LPFC_QUE_INFO_GET_BUF_SIZE - len, "Truncated ...\n");
+out:
spin_unlock_irq(&phba->hbalock);
return simple_read_from_buffer(buf, nbytes, ppos, pbuffer, len);
}
struct lpfc_hba *phba = (struct lpfc_hba *)debug->i_private;
uint32_t qidx, quetp, queid, index, count, offset, value;
uint32_t *pentry;
- struct lpfc_queue *pque;
+ struct lpfc_queue *pque, *qp;
int rc;
/* This is a user write operation */
case LPFC_IDIAG_EQ:
/* HBA event queue */
if (phba->sli4_hba.hba_eq) {
- for (qidx = 0; qidx < phba->cfg_fcp_io_channel;
- qidx++) {
- if (phba->sli4_hba.hba_eq[qidx] &&
- phba->sli4_hba.hba_eq[qidx]->queue_id ==
- queid) {
+ for (qidx = 0; qidx < phba->io_channel_irqs; qidx++) {
+ qp = phba->sli4_hba.hba_eq[qidx];
+ if (qp && qp->queue_id == queid) {
/* Sanity check */
- rc = lpfc_idiag_que_param_check(
- phba->sli4_hba.hba_eq[qidx],
+ rc = lpfc_idiag_que_param_check(qp,
index, count);
if (rc)
goto error_out;
- idiag.ptr_private =
- phba->sli4_hba.hba_eq[qidx];
+ idiag.ptr_private = qp;
goto pass_check;
}
}
idiag.ptr_private = phba->sli4_hba.els_cq;
goto pass_check;
}
+ /* NVME LS complete queue */
+ if (phba->sli4_hba.nvmels_cq &&
+ phba->sli4_hba.nvmels_cq->queue_id == queid) {
+ /* Sanity check */
+ rc = lpfc_idiag_que_param_check(
+ phba->sli4_hba.nvmels_cq, index, count);
+ if (rc)
+ goto error_out;
+ idiag.ptr_private = phba->sli4_hba.nvmels_cq;
+ goto pass_check;
+ }
+ /* NVME LS complete queue */
+ if (phba->sli4_hba.nvmels_cq &&
+ phba->sli4_hba.nvmels_cq->queue_id == queid) {
+ /* Sanity check */
+ rc = lpfc_idiag_que_param_check(
+ phba->sli4_hba.nvmels_cq, index, count);
+ if (rc)
+ goto error_out;
+ idiag.ptr_private = phba->sli4_hba.nvmels_cq;
+ goto pass_check;
+ }
/* FCP complete queue */
if (phba->sli4_hba.fcp_cq) {
+ for (qidx = 0; qidx < phba->cfg_fcp_io_channel;
+ qidx++) {
+ qp = phba->sli4_hba.fcp_cq[qidx];
+ if (qp && qp->queue_id == queid) {
+ /* Sanity check */
+ rc = lpfc_idiag_que_param_check(
+ qp, index, count);
+ if (rc)
+ goto error_out;
+ idiag.ptr_private = qp;
+ goto pass_check;
+ }
+ }
+ }
+ /* NVME complete queue */
+ if (phba->sli4_hba.nvme_cq) {
qidx = 0;
do {
- if (phba->sli4_hba.fcp_cq[qidx] &&
- phba->sli4_hba.fcp_cq[qidx]->queue_id ==
+ if (phba->sli4_hba.nvme_cq[qidx] &&
+ phba->sli4_hba.nvme_cq[qidx]->queue_id ==
queid) {
/* Sanity check */
rc = lpfc_idiag_que_param_check(
- phba->sli4_hba.fcp_cq[qidx],
+ phba->sli4_hba.nvme_cq[qidx],
index, count);
if (rc)
goto error_out;
idiag.ptr_private =
- phba->sli4_hba.fcp_cq[qidx];
+ phba->sli4_hba.nvme_cq[qidx];
goto pass_check;
}
- } while (++qidx < phba->cfg_fcp_io_channel);
+ } while (++qidx < phba->cfg_nvme_io_channel);
}
goto error_out;
break;
idiag.ptr_private = phba->sli4_hba.els_wq;
goto pass_check;
}
+ /* NVME LS work queue */
+ if (phba->sli4_hba.nvmels_wq &&
+ phba->sli4_hba.nvmels_wq->queue_id == queid) {
+ /* Sanity check */
+ rc = lpfc_idiag_que_param_check(
+ phba->sli4_hba.nvmels_wq, index, count);
+ if (rc)
+ goto error_out;
+ idiag.ptr_private = phba->sli4_hba.nvmels_wq;
+ goto pass_check;
+ }
+ /* NVME LS work queue */
+ if (phba->sli4_hba.nvmels_wq &&
+ phba->sli4_hba.nvmels_wq->queue_id == queid) {
+ /* Sanity check */
+ rc = lpfc_idiag_que_param_check(
+ phba->sli4_hba.nvmels_wq, index, count);
+ if (rc)
+ goto error_out;
+ idiag.ptr_private = phba->sli4_hba.nvmels_wq;
+ goto pass_check;
+ }
/* FCP work queue */
if (phba->sli4_hba.fcp_wq) {
for (qidx = 0; qidx < phba->cfg_fcp_io_channel;
+ qidx++) {
+ qp = phba->sli4_hba.fcp_wq[qidx];
+ if (qp && qp->queue_id == queid) {
+ /* Sanity check */
+ rc = lpfc_idiag_que_param_check(
+ qp, index, count);
+ if (rc)
+ goto error_out;
+ idiag.ptr_private = qp;
+ goto pass_check;
+ }
+ }
+ }
+ /* NVME work queue */
+ if (phba->sli4_hba.nvme_wq) {
+ for (qidx = 0; qidx < phba->cfg_nvme_io_channel;
+ qidx++) {
+ qp = phba->sli4_hba.nvme_wq[qidx];
+ if (qp && qp->queue_id == queid) {
+ /* Sanity check */
+ rc = lpfc_idiag_que_param_check(
+ qp, index, count);
+ if (rc)
+ goto error_out;
+ idiag.ptr_private = qp;
+ goto pass_check;
+ }
+ }
+ }
+
+ /* NVME work queues */
+ if (phba->sli4_hba.nvme_wq) {
+ for (qidx = 0; qidx < phba->cfg_nvme_io_channel;
qidx++) {
- if (!phba->sli4_hba.fcp_wq[qidx])
+ if (!phba->sli4_hba.nvme_wq[qidx])
continue;
- if (phba->sli4_hba.fcp_wq[qidx]->queue_id ==
+ if (phba->sli4_hba.nvme_wq[qidx]->queue_id ==
queid) {
/* Sanity check */
rc = lpfc_idiag_que_param_check(
- phba->sli4_hba.fcp_wq[qidx],
+ phba->sli4_hba.nvme_wq[qidx],
index, count);
if (rc)
goto error_out;
idiag.ptr_private =
- phba->sli4_hba.fcp_wq[qidx];
+ phba->sli4_hba.nvme_wq[qidx];
goto pass_check;
}
}
.release = lpfc_debugfs_release,
};
+#undef lpfc_debugfs_op_nvmestat
+static const struct file_operations lpfc_debugfs_op_nvmestat = {
+ .owner = THIS_MODULE,
+ .open = lpfc_debugfs_nvmestat_open,
+ .llseek = lpfc_debugfs_lseek,
+ .read = lpfc_debugfs_read,
+ .write = lpfc_debugfs_nvmestat_write,
+ .release = lpfc_debugfs_release,
+};
+
+#undef lpfc_debugfs_op_nvmektime
+static const struct file_operations lpfc_debugfs_op_nvmektime = {
+ .owner = THIS_MODULE,
+ .open = lpfc_debugfs_nvmektime_open,
+ .llseek = lpfc_debugfs_lseek,
+ .read = lpfc_debugfs_read,
+ .write = lpfc_debugfs_nvmektime_write,
+ .release = lpfc_debugfs_release,
+};
+
+#undef lpfc_debugfs_op_nvmeio_trc
+static const struct file_operations lpfc_debugfs_op_nvmeio_trc = {
+ .owner = THIS_MODULE,
+ .open = lpfc_debugfs_nvmeio_trc_open,
+ .llseek = lpfc_debugfs_lseek,
+ .read = lpfc_debugfs_read,
+ .write = lpfc_debugfs_nvmeio_trc_write,
+ .release = lpfc_debugfs_release,
+};
+
+#undef lpfc_debugfs_op_cpucheck
+static const struct file_operations lpfc_debugfs_op_cpucheck = {
+ .owner = THIS_MODULE,
+ .open = lpfc_debugfs_cpucheck_open,
+ .llseek = lpfc_debugfs_lseek,
+ .read = lpfc_debugfs_read,
+ .write = lpfc_debugfs_cpucheck_write,
+ .release = lpfc_debugfs_release,
+};
+
#undef lpfc_debugfs_op_dumpData
static const struct file_operations lpfc_debugfs_op_dumpData = {
.owner = THIS_MODULE,
if ((mbox_tp == mbox_rd) && (dma_tp == dma_mbox)) {
if (*mbx_dump_map & LPFC_BSG_DMP_MBX_RD_MBX) {
do_dump |= LPFC_BSG_DMP_MBX_RD_MBX;
- printk(KERN_ERR "\nRead mbox command (x%x), "
+ pr_err("\nRead mbox command (x%x), "
"nemb:0x%x, extbuf_cnt:%d:\n",
sta_tp, nemb_tp, ext_buf);
}
if ((mbox_tp == mbox_rd) && (dma_tp == dma_ebuf)) {
if (*mbx_dump_map & LPFC_BSG_DMP_MBX_RD_BUF) {
do_dump |= LPFC_BSG_DMP_MBX_RD_BUF;
- printk(KERN_ERR "\nRead mbox buffer (x%x), "
+ pr_err("\nRead mbox buffer (x%x), "
"nemb:0x%x, extbuf_seq:%d:\n",
sta_tp, nemb_tp, ext_buf);
}
if ((mbox_tp == mbox_wr) && (dma_tp == dma_mbox)) {
if (*mbx_dump_map & LPFC_BSG_DMP_MBX_WR_MBX) {
do_dump |= LPFC_BSG_DMP_MBX_WR_MBX;
- printk(KERN_ERR "\nWrite mbox command (x%x), "
+ pr_err("\nWrite mbox command (x%x), "
"nemb:0x%x, extbuf_cnt:%d:\n",
sta_tp, nemb_tp, ext_buf);
}
if ((mbox_tp == mbox_wr) && (dma_tp == dma_ebuf)) {
if (*mbx_dump_map & LPFC_BSG_DMP_MBX_WR_BUF) {
do_dump |= LPFC_BSG_DMP_MBX_WR_BUF;
- printk(KERN_ERR "\nWrite mbox buffer (x%x), "
+ pr_err("\nWrite mbox buffer (x%x), "
"nemb:0x%x, extbuf_seq:%d:\n",
sta_tp, nemb_tp, ext_buf);
}
for (i = 0; i < *mbx_word_cnt; i++) {
if (!(i % 8)) {
if (i != 0)
- printk(KERN_ERR "%s\n", line_buf);
+ pr_err("%s\n", line_buf);
len = 0;
len += snprintf(line_buf+len,
LPFC_MBX_ACC_LBUF_SZ-len,
pword++;
}
if ((i - 1) % 8)
- printk(KERN_ERR "%s\n", line_buf);
+ pr_err("%s\n", line_buf);
(*mbx_dump_cnt)--;
}
/* dump buffer content */
if (*mbx_dump_map & LPFC_MBX_DMP_MBX_WORD) {
- printk(KERN_ERR "Mailbox command:0x%x dump by word:\n",
+ pr_err("Mailbox command:0x%x dump by word:\n",
pmbox->mbxCommand);
pword = (uint32_t *)pmbox;
for (i = 0; i < *mbx_word_cnt; i++) {
if (!(i % 8)) {
if (i != 0)
- printk(KERN_ERR "%s\n", line_buf);
+ pr_err("%s\n", line_buf);
len = 0;
memset(line_buf, 0, LPFC_MBX_ACC_LBUF_SZ);
len += snprintf(line_buf+len,
pword++;
}
if ((i - 1) % 8)
- printk(KERN_ERR "%s\n", line_buf);
- printk(KERN_ERR "\n");
+ pr_err("%s\n", line_buf);
+ pr_err("\n");
}
if (*mbx_dump_map & LPFC_MBX_DMP_MBX_BYTE) {
- printk(KERN_ERR "Mailbox command:0x%x dump by byte:\n",
+ pr_err("Mailbox command:0x%x dump by byte:\n",
pmbox->mbxCommand);
pbyte = (uint8_t *)pmbox;
for (i = 0; i < *mbx_word_cnt; i++) {
if (!(i % 8)) {
if (i != 0)
- printk(KERN_ERR "%s\n", line_buf);
+ pr_err("%s\n", line_buf);
len = 0;
memset(line_buf, 0, LPFC_MBX_ACC_LBUF_SZ);
len += snprintf(line_buf+len,
LPFC_MBX_ACC_LBUF_SZ-len, " ");
}
if ((i - 1) % 8)
- printk(KERN_ERR "%s\n", line_buf);
- printk(KERN_ERR "\n");
+ pr_err("%s\n", line_buf);
+ pr_err("\n");
}
(*mbx_dump_cnt)--;
i++;
}
lpfc_debugfs_max_slow_ring_trc = (1 << i);
- printk(KERN_ERR
- "lpfc_debugfs_max_disc_trc changed to "
+ pr_err("lpfc_debugfs_max_disc_trc changed to "
"%d\n", lpfc_debugfs_max_disc_trc);
}
}
(sizeof(struct lpfc_debugfs_trc) *
lpfc_debugfs_max_slow_ring_trc));
}
+
+ snprintf(name, sizeof(name), "nvmeio_trc");
+ phba->debug_nvmeio_trc =
+ debugfs_create_file(name, 0644,
+ phba->hba_debugfs_root,
+ phba, &lpfc_debugfs_op_nvmeio_trc);
+ if (!phba->debug_nvmeio_trc) {
+ lpfc_printf_vlog(vport, KERN_ERR, LOG_INIT,
+ "0574 No create debugfs nvmeio_trc\n");
+ goto debug_failed;
+ }
+
+ atomic_set(&phba->nvmeio_trc_cnt, 0);
+ if (lpfc_debugfs_max_nvmeio_trc) {
+ num = lpfc_debugfs_max_nvmeio_trc - 1;
+ if (num & lpfc_debugfs_max_disc_trc) {
+ /* Change to be a power of 2 */
+ num = lpfc_debugfs_max_nvmeio_trc;
+ i = 0;
+ while (num > 1) {
+ num = num >> 1;
+ i++;
+ }
+ lpfc_debugfs_max_nvmeio_trc = (1 << i);
+ lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+ "0575 lpfc_debugfs_max_nvmeio_trc "
+ "changed to %d\n",
+ lpfc_debugfs_max_nvmeio_trc);
+ }
+ phba->nvmeio_trc_size = lpfc_debugfs_max_nvmeio_trc;
+
+ /* Allocate trace buffer and initialize */
+ phba->nvmeio_trc = kmalloc(
+ (sizeof(struct lpfc_debugfs_nvmeio_trc) *
+ phba->nvmeio_trc_size), GFP_KERNEL);
+
+ if (!phba->nvmeio_trc) {
+ lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+ "0576 Cannot create debugfs "
+ "nvmeio_trc buffer\n");
+ goto nvmeio_off;
+ }
+ memset(phba->nvmeio_trc, 0,
+ (sizeof(struct lpfc_debugfs_nvmeio_trc) *
+ phba->nvmeio_trc_size));
+ phba->nvmeio_trc_on = 1;
+ phba->nvmeio_trc_output_idx = 0;
+ phba->nvmeio_trc = NULL;
+ } else {
+nvmeio_off:
+ phba->nvmeio_trc_size = 0;
+ phba->nvmeio_trc_on = 0;
+ phba->nvmeio_trc_output_idx = 0;
+ phba->nvmeio_trc = NULL;
+ }
}
snprintf(name, sizeof(name), "vport%d", vport->vpi);
i++;
}
lpfc_debugfs_max_disc_trc = (1 << i);
- printk(KERN_ERR
- "lpfc_debugfs_max_disc_trc changed to %d\n",
+ pr_err("lpfc_debugfs_max_disc_trc changed to %d\n",
lpfc_debugfs_max_disc_trc);
}
}
goto debug_failed;
}
+ snprintf(name, sizeof(name), "nvmestat");
+ vport->debug_nvmestat =
+ debugfs_create_file(name, 0644,
+ vport->vport_debugfs_root,
+ vport, &lpfc_debugfs_op_nvmestat);
+ if (!vport->debug_nvmestat) {
+ lpfc_printf_vlog(vport, KERN_ERR, LOG_INIT,
+ "0811 Cannot create debugfs nvmestat\n");
+ goto debug_failed;
+ }
+
+ snprintf(name, sizeof(name), "nvmektime");
+ vport->debug_nvmektime =
+ debugfs_create_file(name, 0644,
+ vport->vport_debugfs_root,
+ vport, &lpfc_debugfs_op_nvmektime);
+ if (!vport->debug_nvmektime) {
+ lpfc_printf_vlog(vport, KERN_ERR, LOG_INIT,
+ "0815 Cannot create debugfs nvmektime\n");
+ goto debug_failed;
+ }
+
+ snprintf(name, sizeof(name), "cpucheck");
+ vport->debug_cpucheck =
+ debugfs_create_file(name, 0644,
+ vport->vport_debugfs_root,
+ vport, &lpfc_debugfs_op_cpucheck);
+ if (!vport->debug_cpucheck) {
+ lpfc_printf_vlog(vport, KERN_ERR, LOG_INIT,
+ "0819 Cannot create debugfs cpucheck\n");
+ goto debug_failed;
+ }
+
/*
* The following section is for additional directories/files for the
* physical port.
kfree(vport->disc_trc);
vport->disc_trc = NULL;
}
- if (vport->debug_disc_trc) {
- debugfs_remove(vport->debug_disc_trc); /* discovery_trace */
- vport->debug_disc_trc = NULL;
- }
- if (vport->debug_nodelist) {
- debugfs_remove(vport->debug_nodelist); /* nodelist */
- vport->debug_nodelist = NULL;
- }
+
+ debugfs_remove(vport->debug_disc_trc); /* discovery_trace */
+ vport->debug_disc_trc = NULL;
+
+ debugfs_remove(vport->debug_nodelist); /* nodelist */
+ vport->debug_nodelist = NULL;
+
+ debugfs_remove(vport->debug_nvmestat); /* nvmestat */
+ vport->debug_nvmestat = NULL;
+
+ debugfs_remove(vport->debug_nvmektime); /* nvmektime */
+ vport->debug_nvmektime = NULL;
+
+ debugfs_remove(vport->debug_cpucheck); /* cpucheck */
+ vport->debug_cpucheck = NULL;
+
if (vport->vport_debugfs_root) {
debugfs_remove(vport->vport_debugfs_root); /* vportX */
vport->vport_debugfs_root = NULL;
atomic_dec(&phba->debugfs_vport_count);
}
+
if (atomic_read(&phba->debugfs_vport_count) == 0) {
- if (phba->debug_hbqinfo) {
- debugfs_remove(phba->debug_hbqinfo); /* hbqinfo */
- phba->debug_hbqinfo = NULL;
- }
- if (phba->debug_dumpHBASlim) {
- debugfs_remove(phba->debug_dumpHBASlim); /* HBASlim */
- phba->debug_dumpHBASlim = NULL;
- }
- if (phba->debug_dumpHostSlim) {
- debugfs_remove(phba->debug_dumpHostSlim); /* HostSlim */
- phba->debug_dumpHostSlim = NULL;
- }
- if (phba->debug_dumpData) {
- debugfs_remove(phba->debug_dumpData); /* dumpData */
- phba->debug_dumpData = NULL;
- }
+ debugfs_remove(phba->debug_hbqinfo); /* hbqinfo */
+ phba->debug_hbqinfo = NULL;
- if (phba->debug_dumpDif) {
- debugfs_remove(phba->debug_dumpDif); /* dumpDif */
- phba->debug_dumpDif = NULL;
- }
- if (phba->debug_InjErrLBA) {
- debugfs_remove(phba->debug_InjErrLBA); /* InjErrLBA */
- phba->debug_InjErrLBA = NULL;
- }
- if (phba->debug_InjErrNPortID) { /* InjErrNPortID */
- debugfs_remove(phba->debug_InjErrNPortID);
- phba->debug_InjErrNPortID = NULL;
- }
- if (phba->debug_InjErrWWPN) {
- debugfs_remove(phba->debug_InjErrWWPN); /* InjErrWWPN */
- phba->debug_InjErrWWPN = NULL;
- }
- if (phba->debug_writeGuard) {
- debugfs_remove(phba->debug_writeGuard); /* writeGuard */
- phba->debug_writeGuard = NULL;
- }
- if (phba->debug_writeApp) {
- debugfs_remove(phba->debug_writeApp); /* writeApp */
- phba->debug_writeApp = NULL;
- }
- if (phba->debug_writeRef) {
- debugfs_remove(phba->debug_writeRef); /* writeRef */
- phba->debug_writeRef = NULL;
- }
- if (phba->debug_readGuard) {
- debugfs_remove(phba->debug_readGuard); /* readGuard */
- phba->debug_readGuard = NULL;
- }
- if (phba->debug_readApp) {
- debugfs_remove(phba->debug_readApp); /* readApp */
- phba->debug_readApp = NULL;
- }
- if (phba->debug_readRef) {
- debugfs_remove(phba->debug_readRef); /* readRef */
- phba->debug_readRef = NULL;
- }
+ debugfs_remove(phba->debug_dumpHBASlim); /* HBASlim */
+ phba->debug_dumpHBASlim = NULL;
+
+ debugfs_remove(phba->debug_dumpHostSlim); /* HostSlim */
+ phba->debug_dumpHostSlim = NULL;
+
+ debugfs_remove(phba->debug_dumpData); /* dumpData */
+ phba->debug_dumpData = NULL;
+
+ debugfs_remove(phba->debug_dumpDif); /* dumpDif */
+ phba->debug_dumpDif = NULL;
+
+ debugfs_remove(phba->debug_InjErrLBA); /* InjErrLBA */
+ phba->debug_InjErrLBA = NULL;
+
+ debugfs_remove(phba->debug_InjErrNPortID);
+ phba->debug_InjErrNPortID = NULL;
+
+ debugfs_remove(phba->debug_InjErrWWPN); /* InjErrWWPN */
+ phba->debug_InjErrWWPN = NULL;
+
+ debugfs_remove(phba->debug_writeGuard); /* writeGuard */
+ phba->debug_writeGuard = NULL;
+
+ debugfs_remove(phba->debug_writeApp); /* writeApp */
+ phba->debug_writeApp = NULL;
+
+ debugfs_remove(phba->debug_writeRef); /* writeRef */
+ phba->debug_writeRef = NULL;
+
+ debugfs_remove(phba->debug_readGuard); /* readGuard */
+ phba->debug_readGuard = NULL;
+
+ debugfs_remove(phba->debug_readApp); /* readApp */
+ phba->debug_readApp = NULL;
+
+ debugfs_remove(phba->debug_readRef); /* readRef */
+ phba->debug_readRef = NULL;
if (phba->slow_ring_trc) {
kfree(phba->slow_ring_trc);
phba->slow_ring_trc = NULL;
}
- if (phba->debug_slow_ring_trc) {
- /* slow_ring_trace */
- debugfs_remove(phba->debug_slow_ring_trc);
- phba->debug_slow_ring_trc = NULL;
- }
+
+ /* slow_ring_trace */
+ debugfs_remove(phba->debug_slow_ring_trc);
+ phba->debug_slow_ring_trc = NULL;
+
+ debugfs_remove(phba->debug_nvmeio_trc);
+ phba->debug_nvmeio_trc = NULL;
+
+ kfree(phba->nvmeio_trc);
+ phba->nvmeio_trc = NULL;
/*
* iDiag release
*/
if (phba->sli_rev == LPFC_SLI_REV4) {
- if (phba->idiag_ext_acc) {
- /* iDiag extAcc */
- debugfs_remove(phba->idiag_ext_acc);
- phba->idiag_ext_acc = NULL;
- }
- if (phba->idiag_mbx_acc) {
- /* iDiag mbxAcc */
- debugfs_remove(phba->idiag_mbx_acc);
- phba->idiag_mbx_acc = NULL;
- }
- if (phba->idiag_ctl_acc) {
- /* iDiag ctlAcc */
- debugfs_remove(phba->idiag_ctl_acc);
- phba->idiag_ctl_acc = NULL;
- }
- if (phba->idiag_drb_acc) {
- /* iDiag drbAcc */
- debugfs_remove(phba->idiag_drb_acc);
- phba->idiag_drb_acc = NULL;
- }
- if (phba->idiag_que_acc) {
- /* iDiag queAcc */
- debugfs_remove(phba->idiag_que_acc);
- phba->idiag_que_acc = NULL;
- }
- if (phba->idiag_que_info) {
- /* iDiag queInfo */
- debugfs_remove(phba->idiag_que_info);
- phba->idiag_que_info = NULL;
- }
- if (phba->idiag_bar_acc) {
- /* iDiag barAcc */
- debugfs_remove(phba->idiag_bar_acc);
- phba->idiag_bar_acc = NULL;
- }
- if (phba->idiag_pci_cfg) {
- /* iDiag pciCfg */
- debugfs_remove(phba->idiag_pci_cfg);
- phba->idiag_pci_cfg = NULL;
- }
+ /* iDiag extAcc */
+ debugfs_remove(phba->idiag_ext_acc);
+ phba->idiag_ext_acc = NULL;
+
+ /* iDiag mbxAcc */
+ debugfs_remove(phba->idiag_mbx_acc);
+ phba->idiag_mbx_acc = NULL;
+
+ /* iDiag ctlAcc */
+ debugfs_remove(phba->idiag_ctl_acc);
+ phba->idiag_ctl_acc = NULL;
+
+ /* iDiag drbAcc */
+ debugfs_remove(phba->idiag_drb_acc);
+ phba->idiag_drb_acc = NULL;
+
+ /* iDiag queAcc */
+ debugfs_remove(phba->idiag_que_acc);
+ phba->idiag_que_acc = NULL;
+
+ /* iDiag queInfo */
+ debugfs_remove(phba->idiag_que_info);
+ phba->idiag_que_info = NULL;
+
+ /* iDiag barAcc */
+ debugfs_remove(phba->idiag_bar_acc);
+ phba->idiag_bar_acc = NULL;
+
+ /* iDiag pciCfg */
+ debugfs_remove(phba->idiag_pci_cfg);
+ phba->idiag_pci_cfg = NULL;
/* Finally remove the iDiag debugfs root */
- if (phba->idiag_root) {
- /* iDiag root */
- debugfs_remove(phba->idiag_root);
- phba->idiag_root = NULL;
- }
+ debugfs_remove(phba->idiag_root);
+ phba->idiag_root = NULL;
}
if (phba->hba_debugfs_root) {
atomic_dec(&lpfc_debugfs_hba_count);
}
- if (atomic_read(&lpfc_debugfs_hba_count) == 0) {
- debugfs_remove(lpfc_debugfs_root); /* lpfc */
- lpfc_debugfs_root = NULL;
- }
+ debugfs_remove(lpfc_debugfs_root); /* lpfc */
+ lpfc_debugfs_root = NULL;
}
#endif
return;
void
lpfc_debug_dump_all_queues(struct lpfc_hba *phba)
{
- int fcp_wqidx;
+ int idx;
/*
* Dump Work Queues (WQs)
*/
- lpfc_debug_dump_mbx_wq(phba);
- lpfc_debug_dump_els_wq(phba);
+ lpfc_debug_dump_wq(phba, DUMP_MBX, 0);
+ lpfc_debug_dump_wq(phba, DUMP_ELS, 0);
+ lpfc_debug_dump_wq(phba, DUMP_NVMELS, 0);
- for (fcp_wqidx = 0; fcp_wqidx < phba->cfg_fcp_io_channel; fcp_wqidx++)
- lpfc_debug_dump_fcp_wq(phba, fcp_wqidx);
+ for (idx = 0; idx < phba->cfg_fcp_io_channel; idx++)
+ lpfc_debug_dump_wq(phba, DUMP_FCP, idx);
+
+ for (idx = 0; idx < phba->cfg_nvme_io_channel; idx++)
+ lpfc_debug_dump_wq(phba, DUMP_NVME, idx);
lpfc_debug_dump_hdr_rq(phba);
lpfc_debug_dump_dat_rq(phba);
/*
* Dump Complete Queues (CQs)
*/
- lpfc_debug_dump_mbx_cq(phba);
- lpfc_debug_dump_els_cq(phba);
+ lpfc_debug_dump_cq(phba, DUMP_MBX, 0);
+ lpfc_debug_dump_cq(phba, DUMP_ELS, 0);
+ lpfc_debug_dump_cq(phba, DUMP_NVMELS, 0);
+
+ for (idx = 0; idx < phba->cfg_fcp_io_channel; idx++)
+ lpfc_debug_dump_cq(phba, DUMP_FCP, idx);
- for (fcp_wqidx = 0; fcp_wqidx < phba->cfg_fcp_io_channel; fcp_wqidx++)
- lpfc_debug_dump_fcp_cq(phba, fcp_wqidx);
+ for (idx = 0; idx < phba->cfg_nvme_io_channel; idx++)
+ lpfc_debug_dump_cq(phba, DUMP_NVME, idx);
/*
* Dump Event Queues (EQs)
*/
- for (fcp_wqidx = 0; fcp_wqidx < phba->cfg_fcp_io_channel; fcp_wqidx++)
- lpfc_debug_dump_hba_eq(phba, fcp_wqidx);
+ for (idx = 0; idx < phba->io_channel_irqs; idx++)
+ lpfc_debug_dump_hba_eq(phba, idx);
}
/*******************************************************************
* This file is part of the Emulex Linux Device Driver for *
* Fibre Channel Host Bus Adapters. *
+ * Copyright (C) 2017 Broadcom. All Rights Reserved. The term *
+ * “Broadcom” refers to Broadcom Limited and/or its subsidiaries. *
* Copyright (C) 2007-2011 Emulex. All rights reserved. *
* EMULEX and SLI are trademarks of Emulex. *
- * www.emulex.com *
+ * www.broadcom.com *
* *
* This program is free software; you can redistribute it and/or *
* modify it under the terms of version 2 of the GNU General *
/* hbqinfo output buffer size */
#define LPFC_HBQINFO_SIZE 8192
+enum {
+ DUMP_FCP,
+ DUMP_NVME,
+ DUMP_MBX,
+ DUMP_ELS,
+ DUMP_NVMELS,
+};
+
+/* nvmestat output buffer size */
+#define LPFC_NVMESTAT_SIZE 8192
+#define LPFC_NVMEKTIME_SIZE 8192
+#define LPFC_CPUCHECK_SIZE 8192
+#define LPFC_NVMEIO_TRC_SIZE 8192
+
+#define LPFC_DEBUG_OUT_LINE_SZ 80
+
/*
* For SLI4 iDiag debugfs diagnostics tool
*/
#define SIZE_U16 sizeof(uint16_t)
#define SIZE_U32 sizeof(uint32_t)
+#define lpfc_nvmeio_data(phba, fmt, arg...) \
+ { \
+ if (phba->nvmeio_trc_on) \
+ lpfc_debugfs_nvme_trc(phba, fmt, ##arg); \
+ }
+
struct lpfc_debug {
char *i_private;
char op;
unsigned long jif;
};
+struct lpfc_debugfs_nvmeio_trc {
+ char *fmt;
+ uint16_t data1;
+ uint16_t data2;
+ uint32_t data3;
+};
+
struct lpfc_idiag_offset {
uint32_t last_rd;
};
}
/**
- * lpfc_debug_dump_fcp_wq - dump all entries from a fcp work queue
+ * lpfc_debug_dump_wq - dump all entries from the fcp or nvme work queue
* @phba: Pointer to HBA context object.
- * @fcp_wqidx: Index to a FCP work queue.
+ * @wqidx: Index to a FCP or NVME work queue.
*
- * This function dumps all entries from a FCP work queue specified by the
- * @fcp_wqidx.
+ * This function dumps all entries from a FCP or NVME work queue specified
+ * by the wqidx.
**/
static inline void
-lpfc_debug_dump_fcp_wq(struct lpfc_hba *phba, int fcp_wqidx)
+lpfc_debug_dump_wq(struct lpfc_hba *phba, int qtype, int wqidx)
{
- /* sanity check */
- if (fcp_wqidx >= phba->cfg_fcp_io_channel)
+ struct lpfc_queue *wq;
+ char *qtypestr;
+
+ if (qtype == DUMP_FCP) {
+ wq = phba->sli4_hba.fcp_wq[wqidx];
+ qtypestr = "FCP";
+ } else if (qtype == DUMP_NVME) {
+ wq = phba->sli4_hba.nvme_wq[wqidx];
+ qtypestr = "NVME";
+ } else if (qtype == DUMP_MBX) {
+ wq = phba->sli4_hba.mbx_wq;
+ qtypestr = "MBX";
+ } else if (qtype == DUMP_ELS) {
+ wq = phba->sli4_hba.els_wq;
+ qtypestr = "ELS";
+ } else if (qtype == DUMP_NVMELS) {
+ wq = phba->sli4_hba.nvmels_wq;
+ qtypestr = "NVMELS";
+ } else
return;
- printk(KERN_ERR "FCP WQ: WQ[Idx:%d|Qid:%d]\n",
- fcp_wqidx, phba->sli4_hba.fcp_wq[fcp_wqidx]->queue_id);
- lpfc_debug_dump_q(phba->sli4_hba.fcp_wq[fcp_wqidx]);
+ if (qtype == DUMP_FCP || qtype == DUMP_NVME)
+ pr_err("%s WQ: WQ[Idx:%d|Qid:%d]\n",
+ qtypestr, wqidx, wq->queue_id);
+ else
+ pr_err("%s WQ: WQ[Qid:%d]\n",
+ qtypestr, wq->queue_id);
+
+ lpfc_debug_dump_q(wq);
}
/**
- * lpfc_debug_dump_fcp_cq - dump all entries from a fcp work queue's cmpl queue
+ * lpfc_debug_dump_cq - dump all entries from a fcp or nvme work queue's
+ * cmpl queue
* @phba: Pointer to HBA context object.
- * @fcp_wqidx: Index to a FCP work queue.
+ * @wqidx: Index to a FCP work queue.
*
- * This function dumps all entries from a FCP complete queue which is
- * associated to the FCP work queue specified by the @fcp_wqidx.
+ * This function dumps all entries from a FCP or NVME completion queue
+ * which is associated to the work queue specified by the @wqidx.
**/
static inline void
-lpfc_debug_dump_fcp_cq(struct lpfc_hba *phba, int fcp_wqidx)
+lpfc_debug_dump_cq(struct lpfc_hba *phba, int qtype, int wqidx)
{
- int fcp_cqidx, fcp_cqid;
-
- /* sanity check */
- if (fcp_wqidx >= phba->cfg_fcp_io_channel)
+ struct lpfc_queue *wq, *cq, *eq;
+ char *qtypestr;
+ int eqidx;
+
+ /* fcp/nvme wq and cq are 1:1, thus same indexes */
+
+ if (qtype == DUMP_FCP) {
+ wq = phba->sli4_hba.fcp_wq[wqidx];
+ cq = phba->sli4_hba.fcp_cq[wqidx];
+ qtypestr = "FCP";
+ } else if (qtype == DUMP_NVME) {
+ wq = phba->sli4_hba.nvme_wq[wqidx];
+ cq = phba->sli4_hba.nvme_cq[wqidx];
+ qtypestr = "NVME";
+ } else if (qtype == DUMP_MBX) {
+ wq = phba->sli4_hba.mbx_wq;
+ cq = phba->sli4_hba.mbx_cq;
+ qtypestr = "MBX";
+ } else if (qtype == DUMP_ELS) {
+ wq = phba->sli4_hba.els_wq;
+ cq = phba->sli4_hba.els_cq;
+ qtypestr = "ELS";
+ } else if (qtype == DUMP_NVMELS) {
+ wq = phba->sli4_hba.nvmels_wq;
+ cq = phba->sli4_hba.nvmels_cq;
+ qtypestr = "NVMELS";
+ } else
return;
- fcp_cqid = phba->sli4_hba.fcp_wq[fcp_wqidx]->assoc_qid;
- for (fcp_cqidx = 0; fcp_cqidx < phba->cfg_fcp_io_channel; fcp_cqidx++)
- if (phba->sli4_hba.fcp_cq[fcp_cqidx]->queue_id == fcp_cqid)
+ for (eqidx = 0; eqidx < phba->io_channel_irqs; eqidx++) {
+ eq = phba->sli4_hba.hba_eq[eqidx];
+ if (cq->assoc_qid == eq->queue_id)
break;
- if (phba->intr_type == MSIX) {
- if (fcp_cqidx >= phba->cfg_fcp_io_channel)
- return;
- } else {
- if (fcp_cqidx > 0)
- return;
+ }
+ if (eqidx == phba->io_channel_irqs) {
+ pr_err("Couldn't find EQ for CQ. Using EQ[0]\n");
+ eqidx = 0;
+ eq = phba->sli4_hba.hba_eq[0];
}
- printk(KERN_ERR "FCP CQ: WQ[Idx:%d|Qid%d]->CQ[Idx%d|Qid%d]:\n",
- fcp_wqidx, phba->sli4_hba.fcp_wq[fcp_wqidx]->queue_id,
- fcp_cqidx, fcp_cqid);
- lpfc_debug_dump_q(phba->sli4_hba.fcp_cq[fcp_cqidx]);
+ if (qtype == DUMP_FCP || qtype == DUMP_NVME)
+ pr_err("%s CQ: WQ[Idx:%d|Qid%d]->CQ[Idx%d|Qid%d]"
+ "->EQ[Idx:%d|Qid:%d]:\n",
+ qtypestr, wqidx, wq->queue_id, wqidx, cq->queue_id,
+ eqidx, eq->queue_id);
+ else
+ pr_err("%s CQ: WQ[Qid:%d]->CQ[Qid:%d]"
+ "->EQ[Idx:%d|Qid:%d]:\n",
+ qtypestr, wq->queue_id, cq->queue_id,
+ eqidx, eq->queue_id);
+
+ lpfc_debug_dump_q(cq);
}
/**
* associated to the FCP work queue specified by the @fcp_wqidx.
**/
static inline void
-lpfc_debug_dump_hba_eq(struct lpfc_hba *phba, int fcp_wqidx)
+lpfc_debug_dump_hba_eq(struct lpfc_hba *phba, int qidx)
{
- struct lpfc_queue *qdesc;
- int fcp_eqidx, fcp_eqid;
- int fcp_cqidx, fcp_cqid;
+ struct lpfc_queue *qp;
- /* sanity check */
- if (fcp_wqidx >= phba->cfg_fcp_io_channel)
- return;
- fcp_cqid = phba->sli4_hba.fcp_wq[fcp_wqidx]->assoc_qid;
- for (fcp_cqidx = 0; fcp_cqidx < phba->cfg_fcp_io_channel; fcp_cqidx++)
- if (phba->sli4_hba.fcp_cq[fcp_cqidx]->queue_id == fcp_cqid)
- break;
- if (phba->intr_type == MSIX) {
- if (fcp_cqidx >= phba->cfg_fcp_io_channel)
- return;
- } else {
- if (fcp_cqidx > 0)
- return;
- }
+ qp = phba->sli4_hba.hba_eq[qidx];
- fcp_eqidx = fcp_cqidx;
- fcp_eqid = phba->sli4_hba.hba_eq[fcp_eqidx]->queue_id;
- qdesc = phba->sli4_hba.hba_eq[fcp_eqidx];
+ pr_err("EQ[Idx:%d|Qid:%d]\n", qidx, qp->queue_id);
- printk(KERN_ERR "FCP EQ: WQ[Idx:%d|Qid:%d]->CQ[Idx:%d|Qid:%d]->"
- "EQ[Idx:%d|Qid:%d]\n",
- fcp_wqidx, phba->sli4_hba.fcp_wq[fcp_wqidx]->queue_id,
- fcp_cqidx, fcp_cqid, fcp_eqidx, fcp_eqid);
- lpfc_debug_dump_q(qdesc);
-}
-
-/**
- * lpfc_debug_dump_els_wq - dump all entries from the els work queue
- * @phba: Pointer to HBA context object.
- *
- * This function dumps all entries from the ELS work queue.
- **/
-static inline void
-lpfc_debug_dump_els_wq(struct lpfc_hba *phba)
-{
- printk(KERN_ERR "ELS WQ: WQ[Qid:%d]:\n",
- phba->sli4_hba.els_wq->queue_id);
- lpfc_debug_dump_q(phba->sli4_hba.els_wq);
-}
-
-/**
- * lpfc_debug_dump_mbx_wq - dump all entries from the mbox work queue
- * @phba: Pointer to HBA context object.
- *
- * This function dumps all entries from the MBOX work queue.
- **/
-static inline void
-lpfc_debug_dump_mbx_wq(struct lpfc_hba *phba)
-{
- printk(KERN_ERR "MBX WQ: WQ[Qid:%d]\n",
- phba->sli4_hba.mbx_wq->queue_id);
- lpfc_debug_dump_q(phba->sli4_hba.mbx_wq);
+ lpfc_debug_dump_q(qp);
}
/**
lpfc_debug_dump_q(phba->sli4_hba.hdr_rq);
}
-/**
- * lpfc_debug_dump_els_cq - dump all entries from the els complete queue
- * @phba: Pointer to HBA context object.
- *
- * This function dumps all entries from the els complete queue.
- **/
-static inline void
-lpfc_debug_dump_els_cq(struct lpfc_hba *phba)
-{
- printk(KERN_ERR "ELS CQ: WQ[Qid:%d]->CQ[Qid:%d]\n",
- phba->sli4_hba.els_wq->queue_id,
- phba->sli4_hba.els_cq->queue_id);
- lpfc_debug_dump_q(phba->sli4_hba.els_cq);
-}
-
-/**
- * lpfc_debug_dump_mbx_cq - dump all entries from the mbox complete queue
- * @phba: Pointer to HBA context object.
- *
- * This function dumps all entries from the mbox complete queue.
- **/
-static inline void
-lpfc_debug_dump_mbx_cq(struct lpfc_hba *phba)
-{
- printk(KERN_ERR "MBX CQ: WQ[Qid:%d]->CQ[Qid:%d]\n",
- phba->sli4_hba.mbx_wq->queue_id,
- phba->sli4_hba.mbx_cq->queue_id);
- lpfc_debug_dump_q(phba->sli4_hba.mbx_cq);
-}
-
/**
* lpfc_debug_dump_wq_by_id - dump all entries from a work queue by queue id
* @phba: Pointer to HBA context object.
if (phba->sli4_hba.fcp_wq[wq_idx]->queue_id == qid)
break;
if (wq_idx < phba->cfg_fcp_io_channel) {
- printk(KERN_ERR "FCP WQ[Idx:%d|Qid:%d]\n", wq_idx, qid);
+ pr_err("FCP WQ[Idx:%d|Qid:%d]\n", wq_idx, qid);
lpfc_debug_dump_q(phba->sli4_hba.fcp_wq[wq_idx]);
return;
}
+ for (wq_idx = 0; wq_idx < phba->cfg_nvme_io_channel; wq_idx++)
+ if (phba->sli4_hba.nvme_wq[wq_idx]->queue_id == qid)
+ break;
+ if (wq_idx < phba->cfg_nvme_io_channel) {
+ pr_err("NVME WQ[Idx:%d|Qid:%d]\n", wq_idx, qid);
+ lpfc_debug_dump_q(phba->sli4_hba.nvme_wq[wq_idx]);
+ return;
+ }
+
if (phba->sli4_hba.els_wq->queue_id == qid) {
- printk(KERN_ERR "ELS WQ[Qid:%d]\n", qid);
+ pr_err("ELS WQ[Qid:%d]\n", qid);
lpfc_debug_dump_q(phba->sli4_hba.els_wq);
+ return;
+ }
+
+ if (phba->sli4_hba.nvmels_wq->queue_id == qid) {
+ pr_err("NVME LS WQ[Qid:%d]\n", qid);
+ lpfc_debug_dump_q(phba->sli4_hba.nvmels_wq);
}
}
static inline void
lpfc_debug_dump_cq_by_id(struct lpfc_hba *phba, int qid)
{
- int cq_idx = 0;
+ int cq_idx;
- do {
+ for (cq_idx = 0; cq_idx < phba->cfg_fcp_io_channel; cq_idx++)
if (phba->sli4_hba.fcp_cq[cq_idx]->queue_id == qid)
break;
- } while (++cq_idx < phba->cfg_fcp_io_channel);
if (cq_idx < phba->cfg_fcp_io_channel) {
- printk(KERN_ERR "FCP CQ[Idx:%d|Qid:%d]\n", cq_idx, qid);
+ pr_err("FCP CQ[Idx:%d|Qid:%d]\n", cq_idx, qid);
lpfc_debug_dump_q(phba->sli4_hba.fcp_cq[cq_idx]);
return;
}
+ for (cq_idx = 0; cq_idx < phba->cfg_nvme_io_channel; cq_idx++)
+ if (phba->sli4_hba.nvme_cq[cq_idx]->queue_id == qid)
+ break;
+
+ if (cq_idx < phba->cfg_nvme_io_channel) {
+ pr_err("NVME CQ[Idx:%d|Qid:%d]\n", cq_idx, qid);
+ lpfc_debug_dump_q(phba->sli4_hba.nvme_cq[cq_idx]);
+ return;
+ }
+
if (phba->sli4_hba.els_cq->queue_id == qid) {
- printk(KERN_ERR "ELS CQ[Qid:%d]\n", qid);
+ pr_err("ELS CQ[Qid:%d]\n", qid);
lpfc_debug_dump_q(phba->sli4_hba.els_cq);
return;
}
+ if (phba->sli4_hba.nvmels_cq->queue_id == qid) {
+ pr_err("NVME LS CQ[Qid:%d]\n", qid);
+ lpfc_debug_dump_q(phba->sli4_hba.nvmels_cq);
+ return;
+ }
+
if (phba->sli4_hba.mbx_cq->queue_id == qid) {
- printk(KERN_ERR "MBX CQ[Qid:%d]\n", qid);
+ pr_err("MBX CQ[Qid:%d]\n", qid);
lpfc_debug_dump_q(phba->sli4_hba.mbx_cq);
}
}
{
int eq_idx;
- for (eq_idx = 0; eq_idx < phba->cfg_fcp_io_channel; eq_idx++) {
+ for (eq_idx = 0; eq_idx < phba->io_channel_irqs; eq_idx++)
if (phba->sli4_hba.hba_eq[eq_idx]->queue_id == qid)
break;
- }
- if (eq_idx < phba->cfg_fcp_io_channel) {
+ if (eq_idx < phba->io_channel_irqs) {
printk(KERN_ERR "FCP EQ[Idx:%d|Qid:%d]\n", eq_idx, qid);
lpfc_debug_dump_q(phba->sli4_hba.hba_eq[eq_idx]);
return;
}
-
}
void lpfc_debug_dump_all_queues(struct lpfc_hba *);
/*******************************************************************
* This file is part of the Emulex Linux Device Driver for *
* Fibre Channel Host Bus Adapters. *
+ * Copyright (C) 2017 Broadcom. All Rights Reserved. The term *
+ * “Broadcom” refers to Broadcom Limited and/or its subsidiaries. *
* Copyright (C) 2004-2013 Emulex. All rights reserved. *
* EMULEX and SLI are trademarks of Emulex. *
- * www.emulex.com *
+ * www.broadcom.com *
* *
* This program is free software; you can redistribute it and/or *
* modify it under the terms of version 2 of the GNU General *
#define NLP_FABRIC 0x4 /* entry rep a Fabric entity */
#define NLP_FCP_TARGET 0x8 /* entry is an FCP target */
#define NLP_FCP_INITIATOR 0x10 /* entry is an FCP Initiator */
+#define NLP_NVME_TARGET 0x20 /* entry is a NVME Target */
+#define NLP_NVME_INITIATOR 0x40 /* entry is a NVME Initiator */
+
+ uint16_t nlp_fc4_type; /* FC types node supports. */
+ /* Assigned from GID_FF, only
+ * FCP (0x8) and NVME (0x28)
+ * supported.
+ */
+#define NLP_FC4_NONE 0x0
+#define NLP_FC4_FCP 0x1 /* FC4 Type FCP (value x8)) */
+#define NLP_FC4_NVME 0x2 /* FC4 TYPE NVME (value x28) */
uint16_t nlp_rpi;
uint16_t nlp_state; /* state transition indicator */
struct timer_list nlp_delayfunc; /* Used for delayed ELS cmds */
struct lpfc_hba *phba;
- struct fc_rport *rport; /* Corresponding FC transport
- port structure */
+ struct fc_rport *rport; /* scsi_transport_fc port structure */
+ struct lpfc_nvme_rport *nrport; /* nvme transport rport struct. */
struct lpfc_vport *vport;
struct lpfc_work_evt els_retry_evt;
struct lpfc_work_evt dev_loss_evt;
unsigned long last_change_time;
unsigned long *active_rrqs_xri_bitmap;
struct lpfc_scsicmd_bkt *lat_data; /* Latency data */
+ uint32_t fc4_prli_sent;
+ uint32_t upcall_flags;
+ uint32_t nvme_fb_size; /* NVME target's supported byte cnt */
+#define NVME_FB_BIT_SHIFT 9 /* PRLI Rsp first burst in 512B units. */
};
struct lpfc_node_rrq {
struct list_head list;
/* Defines for nlp_flag (uint32) */
#define NLP_IGNR_REG_CMPL 0x00000001 /* Rcvd rscn before we cmpl reg login */
#define NLP_REG_LOGIN_SEND 0x00000002 /* sent reglogin to adapter */
+#define NLP_SUPPRESS_RSP 0x00000010 /* Remote NPort supports suppress rsp */
#define NLP_PLOGI_SND 0x00000020 /* sent PLOGI request for this entry */
#define NLP_PRLI_SND 0x00000040 /* sent PRLI request for this entry */
#define NLP_ADISC_SND 0x00000080 /* sent ADISC request for this entry */
/*******************************************************************
* This file is part of the Emulex Linux Device Driver for *
* Fibre Channel Host Bus Adapters. *
+ * Copyright (C) 2017 Broadcom. All Rights Reserved. The term *
+ * “Broadcom” refers to Broadcom Limited and/or its subsidiaries. *
* Copyright (C) 2004-2016 Emulex. All rights reserved. *
* EMULEX and SLI are trademarks of Emulex. *
- * www.emulex.com *
+ * www.broadcom.com *
* Portions Copyright (C) 2004-2005 Christoph Hellwig *
* *
* This program is free software; you can redistribute it and/or *
#include <scsi/scsi_host.h>
#include <scsi/scsi_transport_fc.h>
-
#include "lpfc_hw4.h"
#include "lpfc_hw.h"
#include "lpfc_sli.h"
"0201 Abort outstanding I/O on NPort x%x\n",
Fabric_DID);
- pring = &phba->sli.ring[LPFC_ELS_RING];
+ pring = lpfc_phba_elsring(phba);
/*
* Check the txcmplq for an iocb that matches the nport the driver is
lpfc_plogi_confirm_nport(struct lpfc_hba *phba, uint32_t *prsp,
struct lpfc_nodelist *ndlp)
{
- struct lpfc_vport *vport = ndlp->vport;
+ struct lpfc_vport *vport = ndlp->vport;
struct Scsi_Host *shost = lpfc_shost_from_vport(vport);
struct lpfc_nodelist *new_ndlp;
struct lpfc_rport_data *rdata;
/* PLOGI completes to NPort <nlp_DID> */
lpfc_printf_vlog(vport, KERN_INFO, LOG_ELS,
- "0102 PLOGI completes to NPort x%x "
+ "0102 PLOGI completes to NPort x%06x "
"Data: x%x x%x x%x x%x x%x\n",
- ndlp->nlp_DID, irsp->ulpStatus, irsp->un.ulpWord[4],
- irsp->ulpTimeout, disc, vport->num_disc_nodes);
+ ndlp->nlp_DID, ndlp->nlp_fc4_type,
+ irsp->ulpStatus, irsp->un.ulpWord[4],
+ disc, vport->num_disc_nodes);
+
/* Check to see if link went down during discovery */
if (lpfc_els_chk_latt(vport)) {
spin_lock_irq(shost->host_lock);
sp->cmn.fcphHigh = FC_PH3;
sp->cmn.valid_vendor_ver_level = 0;
- memset(sp->vendorVersion, 0, sizeof(sp->vendorVersion));
+ memset(sp->un.vendorVersion, 0, sizeof(sp->un.vendorVersion));
lpfc_debugfs_disc_trc(vport, LPFC_DISC_TRC_ELS_CMD,
"Issue PLOGI: did:x%x",
did, 0, 0);
+ /* If our firmware supports this feature, convey that
+ * information to the target using the vendor specific field.
+ */
+ if (phba->sli.sli_flag & LPFC_SLI_SUPPRESS_RSP) {
+ sp->cmn.valid_vendor_ver_level = 1;
+ sp->un.vv.vid = cpu_to_be32(LPFC_VV_EMLX_ID);
+ sp->un.vv.flags = cpu_to_be32(LPFC_VV_SUPPRESS_RSP);
+ }
+
phba->fc_stat.elsXmitPLOGI++;
elsiocb->iocb_cmpl = lpfc_cmpl_els_plogi;
ret = lpfc_sli_issue_iocb(phba, LPFC_ELS_RING, elsiocb, 0);
"PRLI cmpl: status:x%x/x%x did:x%x",
irsp->ulpStatus, irsp->un.ulpWord[4],
ndlp->nlp_DID);
+
+ /* Ddriver supports multiple FC4 types. Counters matter. */
+ vport->fc_prli_sent--;
+
/* PRLI completes to NPort <nlp_DID> */
lpfc_printf_vlog(vport, KERN_INFO, LOG_ELS,
- "0103 PRLI completes to NPort x%x "
+ "0103 PRLI completes to NPort x%06x "
"Data: x%x x%x x%x x%x\n",
ndlp->nlp_DID, irsp->ulpStatus, irsp->un.ulpWord[4],
- irsp->ulpTimeout, vport->num_disc_nodes);
+ vport->num_disc_nodes, ndlp->fc4_prli_sent);
- vport->fc_prli_sent--;
/* Check to see if link went down during discovery */
if (lpfc_els_chk_latt(vport))
goto out;
/* Check for retry */
if (lpfc_els_retry(phba, cmdiocb, rspiocb)) {
/* ELS command is being retried */
+ ndlp->fc4_prli_sent--;
goto out;
}
/* PRLI failed */
lpfc_disc_state_machine(vport, ndlp, cmdiocb,
NLP_EVT_CMPL_PRLI);
} else
- /* Good status, call state machine */
+ /* Good status, call state machine. However, if another
+ * PRLI is outstanding, don't call the state machine
+ * because final disposition to Mapped or Unmapped is
+ * completed there.
+ */
lpfc_disc_state_machine(vport, ndlp, cmdiocb,
NLP_EVT_CMPL_PRLI);
+
out:
lpfc_els_free_iocb(phba, cmdiocb);
return;
struct Scsi_Host *shost = lpfc_shost_from_vport(vport);
struct lpfc_hba *phba = vport->phba;
PRLI *npr;
+ struct lpfc_nvme_prli *npr_nvme;
struct lpfc_iocbq *elsiocb;
uint8_t *pcmd;
uint16_t cmdsize;
-
- cmdsize = (sizeof(uint32_t) + sizeof(PRLI));
+ u32 local_nlp_type, elscmd;
+
+ local_nlp_type = ndlp->nlp_fc4_type;
+
+ send_next_prli:
+ if (local_nlp_type & NLP_FC4_FCP) {
+ /* Payload is 4 + 16 = 20 x14 bytes. */
+ cmdsize = (sizeof(uint32_t) + sizeof(PRLI));
+ elscmd = ELS_CMD_PRLI;
+ } else if (local_nlp_type & NLP_FC4_NVME) {
+ /* Payload is 4 + 20 = 24 x18 bytes. */
+ cmdsize = (sizeof(uint32_t) + sizeof(struct lpfc_nvme_prli));
+ elscmd = ELS_CMD_NVMEPRLI;
+ } else {
+ lpfc_printf_vlog(vport, KERN_INFO, LOG_DISCOVERY,
+ "3083 Unknown FC_TYPE x%x ndlp x%06x\n",
+ ndlp->nlp_fc4_type, ndlp->nlp_DID);
+ return 1;
+ }
elsiocb = lpfc_prep_els_iocb(vport, 1, cmdsize, retry, ndlp,
- ndlp->nlp_DID, ELS_CMD_PRLI);
+ ndlp->nlp_DID, elscmd);
if (!elsiocb)
return 1;
pcmd = (uint8_t *) (((struct lpfc_dmabuf *) elsiocb->context2)->virt);
/* For PRLI request, remainder of payload is service parameters */
- memset(pcmd, 0, (sizeof(PRLI) + sizeof(uint32_t)));
- *((uint32_t *) (pcmd)) = ELS_CMD_PRLI;
- pcmd += sizeof(uint32_t);
+ memset(pcmd, 0, cmdsize);
- /* For PRLI, remainder of payload is PRLI parameter page */
- npr = (PRLI *) pcmd;
- /*
- * If our firmware version is 3.20 or later,
- * set the following bits for FC-TAPE support.
- */
- if (phba->vpd.rev.feaLevelHigh >= 0x02) {
- npr->ConfmComplAllowed = 1;
- npr->Retry = 1;
- npr->TaskRetryIdReq = 1;
- }
- npr->estabImagePair = 1;
- npr->readXferRdyDis = 1;
- if (vport->cfg_first_burst_size)
- npr->writeXferRdyDis = 1;
+ if (local_nlp_type & NLP_FC4_FCP) {
+ /* Remainder of payload is FCP PRLI parameter page.
+ * Note: this data structure is defined as
+ * BE/LE in the structure definition so no
+ * byte swap call is made.
+ */
+ *((uint32_t *)(pcmd)) = ELS_CMD_PRLI;
+ pcmd += sizeof(uint32_t);
+ npr = (PRLI *)pcmd;
- /* For FCP support */
- npr->prliType = PRLI_FCP_TYPE;
- npr->initiatorFunc = 1;
+ /*
+ * If our firmware version is 3.20 or later,
+ * set the following bits for FC-TAPE support.
+ */
+ if (phba->vpd.rev.feaLevelHigh >= 0x02) {
+ npr->ConfmComplAllowed = 1;
+ npr->Retry = 1;
+ npr->TaskRetryIdReq = 1;
+ }
+ npr->estabImagePair = 1;
+ npr->readXferRdyDis = 1;
+ if (vport->cfg_first_burst_size)
+ npr->writeXferRdyDis = 1;
+
+ /* For FCP support */
+ npr->prliType = PRLI_FCP_TYPE;
+ npr->initiatorFunc = 1;
+ elsiocb->iocb_flag |= LPFC_PRLI_FCP_REQ;
+
+ /* Remove FCP type - processed. */
+ local_nlp_type &= ~NLP_FC4_FCP;
+ } else if (local_nlp_type & NLP_FC4_NVME) {
+ /* Remainder of payload is NVME PRLI parameter page.
+ * This data structure is the newer definition that
+ * uses bf macros so a byte swap is required.
+ */
+ *((uint32_t *)(pcmd)) = ELS_CMD_NVMEPRLI;
+ pcmd += sizeof(uint32_t);
+ npr_nvme = (struct lpfc_nvme_prli *)pcmd;
+ bf_set(prli_type_code, npr_nvme, PRLI_NVME_TYPE);
+ bf_set(prli_estabImagePair, npr_nvme, 0); /* Should be 0 */
+
+ /* Only initiators request first burst. */
+ if ((phba->cfg_nvme_enable_fb) &&
+ !phba->nvmet_support)
+ bf_set(prli_fba, npr_nvme, 1);
+
+ if (phba->nvmet_support) {
+ bf_set(prli_tgt, npr_nvme, 1);
+ bf_set(prli_disc, npr_nvme, 1);
+
+ } else {
+ bf_set(prli_init, npr_nvme, 1);
+ }
+ npr_nvme->word1 = cpu_to_be32(npr_nvme->word1);
+ npr_nvme->word4 = cpu_to_be32(npr_nvme->word4);
+ elsiocb->iocb_flag |= LPFC_PRLI_NVME_REQ;
+
+ /* Remove NVME type - processed. */
+ local_nlp_type &= ~NLP_FC4_NVME;
+ }
lpfc_debugfs_disc_trc(vport, LPFC_DISC_TRC_ELS_CMD,
"Issue PRLI: did:x%x",
lpfc_els_free_iocb(phba, elsiocb);
return 1;
}
+
+ /* The vport counters are used for lpfc_scan_finished, but
+ * the ndlp is used to track outstanding PRLIs for different
+ * FC4 types.
+ */
vport->fc_prli_sent++;
+ ndlp->fc4_prli_sent++;
+
+ /* The driver supports 2 FC4 types. Make sure
+ * a PRLI is issued for all types before exiting.
+ */
+ if (local_nlp_type & (NLP_FC4_FCP | NLP_FC4_NVME))
+ goto send_next_prli;
+
return 0;
}
if ((vport->fc_flag & FC_PT2PT) &&
!(vport->fc_flag & FC_PT2PT_PLOGI)) {
phba->pport->fc_myDID = 0;
+
+ if ((phba->cfg_enable_fc4_type == LPFC_ENABLE_BOTH) ||
+ (phba->cfg_enable_fc4_type == LPFC_ENABLE_NVME)) {
+ if (phba->nvmet_support)
+ lpfc_nvmet_update_targetport(phba);
+ else
+ lpfc_nvme_update_localport(phba->pport);
+ }
+
mbox = mempool_alloc(phba->mbox_mem_pool, GFP_KERNEL);
if (mbox) {
lpfc_config_link(phba, mbox);
}
break;
case ELS_CMD_PRLI:
+ case ELS_CMD_NVMEPRLI:
if (!lpfc_issue_els_prli(vport, ndlp, retry)) {
ndlp->nlp_prev_state = ndlp->nlp_state;
lpfc_nlp_set_state(vport, ndlp, NLP_STE_PRLI_ISSUE);
break;
}
if ((cmd == ELS_CMD_PLOGI) ||
- (cmd == ELS_CMD_PRLI)) {
+ (cmd == ELS_CMD_PRLI) ||
+ (cmd == ELS_CMD_NVMEPRLI)) {
delay = 1000;
maxretry = lpfc_max_els_tries + 1;
retry = 1;
case LSRJT_LOGICAL_BSY:
if ((cmd == ELS_CMD_PLOGI) ||
- (cmd == ELS_CMD_PRLI)) {
+ (cmd == ELS_CMD_PRLI) ||
+ (cmd == ELS_CMD_NVMEPRLI)) {
delay = 1000;
maxretry = 48;
} else if (cmd == ELS_CMD_FDISC) {
spin_unlock_irq(shost->host_lock);
ndlp->nlp_prev_state = ndlp->nlp_state;
- if (cmd == ELS_CMD_PRLI)
+ if ((cmd == ELS_CMD_PRLI) ||
+ (cmd == ELS_CMD_NVMEPRLI))
lpfc_nlp_set_state(vport, ndlp,
NLP_STE_PRLI_ISSUE);
else
lpfc_issue_els_adisc(vport, ndlp, cmdiocb->retry);
return 1;
case ELS_CMD_PRLI:
+ case ELS_CMD_NVMEPRLI:
ndlp->nlp_prev_state = ndlp->nlp_state;
lpfc_nlp_set_state(vport, ndlp, NLP_STE_PRLI_ISSUE);
lpfc_issue_els_prli(vport, ndlp, cmdiocb->retry);
sizeof(struct serv_parm));
sp->cmn.valid_vendor_ver_level = 0;
- memset(sp->vendorVersion, 0, sizeof(sp->vendorVersion));
+ memset(sp->un.vendorVersion, 0,
+ sizeof(sp->un.vendorVersion));
+
+ /* If our firmware supports this feature, convey that
+ * info to the target using the vendor specific field.
+ */
+ if (phba->sli.sli_flag & LPFC_SLI_SUPPRESS_RSP) {
+ sp->cmn.valid_vendor_ver_level = 1;
+ sp->un.vv.vid = cpu_to_be32(LPFC_VV_EMLX_ID);
+ sp->un.vv.flags =
+ cpu_to_be32(LPFC_VV_SUPPRESS_RSP);
+ }
}
lpfc_debugfs_disc_trc(vport, LPFC_DISC_TRC_ELS_RSP,
{
struct lpfc_hba *phba = vport->phba;
PRLI *npr;
+ struct lpfc_nvme_prli *npr_nvme;
lpfc_vpd_t *vpd;
IOCB_t *icmd;
IOCB_t *oldcmd;
struct lpfc_iocbq *elsiocb;
uint8_t *pcmd;
uint16_t cmdsize;
+ uint32_t prli_fc4_req, *req_payload;
+ struct lpfc_dmabuf *req_buf;
int rc;
+ u32 elsrspcmd;
+
+ /* Need the incoming PRLI payload to determine if the ACC is for an
+ * FC4 or NVME PRLI type. The PRLI type is at word 1.
+ */
+ req_buf = (struct lpfc_dmabuf *)oldiocb->context2;
+ req_payload = (((uint32_t *)req_buf->virt) + 1);
+
+ /* PRLI type payload is at byte 3 for FCP or NVME. */
+ prli_fc4_req = be32_to_cpu(*req_payload);
+ prli_fc4_req = (prli_fc4_req >> 24) & 0xff;
+ lpfc_printf_vlog(vport, KERN_INFO, LOG_ELS,
+ "6127 PRLI_ACC: Req Type x%x, Word1 x%08x\n",
+ prli_fc4_req, *((uint32_t *)req_payload));
+
+ if (prli_fc4_req == PRLI_FCP_TYPE) {
+ cmdsize = sizeof(uint32_t) + sizeof(PRLI);
+ elsrspcmd = (ELS_CMD_ACC | (ELS_CMD_PRLI & ~ELS_RSP_MASK));
+ } else if (prli_fc4_req & PRLI_NVME_TYPE) {
+ cmdsize = sizeof(uint32_t) + sizeof(struct lpfc_nvme_prli);
+ elsrspcmd = (ELS_CMD_ACC | (ELS_CMD_NVMEPRLI & ~ELS_RSP_MASK));
+ } else {
+ return 1;
+ }
- cmdsize = sizeof(uint32_t) + sizeof(PRLI);
elsiocb = lpfc_prep_els_iocb(vport, 0, cmdsize, oldiocb->retry, ndlp,
- ndlp->nlp_DID, (ELS_CMD_ACC | (ELS_CMD_PRLI & ~ELS_RSP_MASK)));
+ ndlp->nlp_DID, elsrspcmd);
if (!elsiocb)
return 1;
ndlp->nlp_DID, ndlp->nlp_flag, ndlp->nlp_state,
ndlp->nlp_rpi);
pcmd = (uint8_t *) (((struct lpfc_dmabuf *) elsiocb->context2)->virt);
+ memset(pcmd, 0, cmdsize);
*((uint32_t *) (pcmd)) = (ELS_CMD_ACC | (ELS_CMD_PRLI & ~ELS_RSP_MASK));
pcmd += sizeof(uint32_t);
/* For PRLI, remainder of payload is PRLI parameter page */
- memset(pcmd, 0, sizeof(PRLI));
-
- npr = (PRLI *) pcmd;
vpd = &phba->vpd;
- /*
- * If the remote port is a target and our firmware version is 3.20 or
- * later, set the following bits for FC-TAPE support.
- */
- if ((ndlp->nlp_type & NLP_FCP_TARGET) &&
- (vpd->rev.feaLevelHigh >= 0x02)) {
- npr->ConfmComplAllowed = 1;
- npr->Retry = 1;
- npr->TaskRetryIdReq = 1;
- }
- npr->acceptRspCode = PRLI_REQ_EXECUTED;
- npr->estabImagePair = 1;
- npr->readXferRdyDis = 1;
- npr->ConfmComplAllowed = 1;
+ if (prli_fc4_req == PRLI_FCP_TYPE) {
+ /*
+ * If the remote port is a target and our firmware version
+ * is 3.20 or later, set the following bits for FC-TAPE
+ * support.
+ */
+ npr = (PRLI *) pcmd;
+ if ((ndlp->nlp_type & NLP_FCP_TARGET) &&
+ (vpd->rev.feaLevelHigh >= 0x02)) {
+ npr->ConfmComplAllowed = 1;
+ npr->Retry = 1;
+ npr->TaskRetryIdReq = 1;
+ }
+ npr->acceptRspCode = PRLI_REQ_EXECUTED;
+ npr->estabImagePair = 1;
+ npr->readXferRdyDis = 1;
+ npr->ConfmComplAllowed = 1;
+ npr->prliType = PRLI_FCP_TYPE;
+ npr->initiatorFunc = 1;
+ } else if (prli_fc4_req & PRLI_NVME_TYPE) {
+ /* Respond with an NVME PRLI Type */
+ npr_nvme = (struct lpfc_nvme_prli *) pcmd;
+ bf_set(prli_type_code, npr_nvme, PRLI_NVME_TYPE);
+ bf_set(prli_estabImagePair, npr_nvme, 0); /* Should be 0 */
+ bf_set(prli_acc_rsp_code, npr_nvme, PRLI_REQ_EXECUTED);
+ if (phba->nvmet_support) {
+ bf_set(prli_tgt, npr_nvme, 1);
+ bf_set(prli_disc, npr_nvme, 1);
+ if (phba->cfg_nvme_enable_fb) {
+ bf_set(prli_fba, npr_nvme, 1);
+
+ /* TBD. Target mode needs to post buffers
+ * that support the configured first burst
+ * byte size.
+ */
+ bf_set(prli_fb_sz, npr_nvme,
+ phba->cfg_nvmet_fb_size);
+ }
+ } else {
+ bf_set(prli_init, npr_nvme, 1);
+ }
- npr->prliType = PRLI_FCP_TYPE;
- npr->initiatorFunc = 1;
+ lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_DISC,
+ "6015 NVME issue PRLI ACC word1 x%08x "
+ "word4 x%08x word5 x%08x flag x%x, "
+ "fcp_info x%x nlp_type x%x\n",
+ npr_nvme->word1, npr_nvme->word4,
+ npr_nvme->word5, ndlp->nlp_flag,
+ ndlp->nlp_fcp_info, ndlp->nlp_type);
+ npr_nvme->word1 = cpu_to_be32(npr_nvme->word1);
+ npr_nvme->word4 = cpu_to_be32(npr_nvme->word4);
+ npr_nvme->word5 = cpu_to_be32(npr_nvme->word5);
+ } else
+ lpfc_printf_vlog(vport, KERN_INFO, LOG_DISCOVERY,
+ "6128 Unknown FC_TYPE x%x x%x ndlp x%06x\n",
+ prli_fc4_req, ndlp->nlp_fc4_type,
+ ndlp->nlp_DID);
lpfc_debugfs_disc_trc(vport, LPFC_DISC_TRC_ELS_RSP,
"Issue ACC PRLI: did:x%x flg:x%x",
**/
static void
lpfc_els_clear_rrq(struct lpfc_vport *vport,
- struct lpfc_iocbq *iocb, struct lpfc_nodelist *ndlp)
+ struct lpfc_iocbq *iocb, struct lpfc_nodelist *ndlp)
{
struct lpfc_hba *phba = vport->phba;
uint8_t *pcmd;
memcpy(desc->opd_info.vendor_name, &page_a0[SSF_VENDOR_NAME], 16);
memcpy(desc->opd_info.model_number, &page_a0[SSF_VENDOR_PN], 16);
memcpy(desc->opd_info.serial_number, &page_a0[SSF_VENDOR_SN], 16);
- memcpy(desc->opd_info.revision, &page_a0[SSF_VENDOR_REV], 2);
+ memcpy(desc->opd_info.revision, &page_a0[SSF_VENDOR_REV], 4);
memcpy(desc->opd_info.date, &page_a0[SSF_DATE_CODE], 8);
desc->length = cpu_to_be32(sizeof(desc->opd_info));
return sizeof(struct fc_rdp_opd_sfp_desc);
memcpy(desc->port_names.wwnn, phba->wwnn,
sizeof(desc->port_names.wwnn));
- memcpy(desc->port_names.wwpn, &phba->wwpn,
+ memcpy(desc->port_names.wwpn, phba->wwpn,
sizeof(desc->port_names.wwpn));
desc->length = cpu_to_be32(sizeof(desc->port_names));
struct ls_rjt stat;
if (phba->sli_rev < LPFC_SLI_REV4 ||
- (bf_get(lpfc_sli_intf_if_type,
- &phba->sli4_hba.sli_intf) !=
- LPFC_SLI_INTF_IF_TYPE_2)) {
+ bf_get(lpfc_sli_intf_if_type, &phba->sli4_hba.sli_intf) !=
+ LPFC_SLI_INTF_IF_TYPE_2) {
rjt_err = LSRJT_UNABLE_TPC;
rjt_expl = LSEXP_REQ_UNSUPPORTED;
goto error;
(ndlp->nlp_state == NLP_STE_UNUSED_NODE) ||
!lpfc_rscn_payload_check(vport, ndlp->nlp_DID))
continue;
+ if (vport->phba->nvmet_support)
+ continue;
lpfc_disc_state_machine(vport, ndlp, NULL,
NLP_EVT_DEVICE_RECOVERY);
lpfc_cancel_retry_delay_tmo(vport, ndlp);
if (ndlp && NLP_CHK_NODE_ACT(ndlp)
&& ndlp->nlp_state == NLP_STE_UNMAPPED_NODE) {
/* Good ndlp, issue CT Request to NameServer */
- if (lpfc_ns_cmd(vport, SLI_CTNS_GID_FT, 0, 0) == 0)
+ vport->gidft_inp = 0;
+ if (lpfc_issue_gidft(vport) == 0)
/* Wait for NameServer query cmpl before we can
- continue */
+ * continue
+ */
return 1;
} else {
/* If login to NameServer does not exist, issue one */
(void) lpfc_check_sparm(vport, ndlp, sp, CLASS3, 1);
-
/*
* If our portname is greater than the remote portname,
* then we initiate Nport login.
timeout = (uint32_t)(phba->fc_ratov << 1);
- pring = &phba->sli.ring[LPFC_ELS_RING];
+ pring = lpfc_phba_elsring(phba);
+
if ((phba->pport->load_flag & FC_UNLOADING))
return;
spin_lock_irq(&phba->hbalock);
spin_unlock_irq(&phba->hbalock);
}
- if (!list_empty(&phba->sli.ring[LPFC_ELS_RING].txcmplq))
+ if (!list_empty(&pring->txcmplq))
if (!(phba->pport->load_flag & FC_UNLOADING))
mod_timer(&vport->els_tmofunc,
jiffies + msecs_to_jiffies(1000 * timeout));
{
LIST_HEAD(abort_list);
struct lpfc_hba *phba = vport->phba;
- struct lpfc_sli_ring *pring = &phba->sli.ring[LPFC_ELS_RING];
+ struct lpfc_sli_ring *pring;
struct lpfc_iocbq *tmp_iocb, *piocb;
IOCB_t *cmd = NULL;
* a working list and release the locks before calling the abort.
*/
spin_lock_irq(&phba->hbalock);
+ pring = lpfc_phba_elsring(phba);
if (phba->sli_rev == LPFC_SLI_REV4)
spin_lock(&pring->ring_lock);
lpfc_els_rcv_fan(vport, elsiocb, ndlp);
break;
case ELS_CMD_PRLI:
+ case ELS_CMD_NVMEPRLI:
lpfc_debugfs_disc_trc(vport, LPFC_DISC_TRC_ELS_UNSOL,
"RCV PRLI: did:x%x/ste:x%x flg:x%x",
did, vport->port_state, ndlp->nlp_flag);
break;
}
- if (atomic_read(&phba->fabric_iocb_count) == 0)
- BUG();
+ BUG_ON(atomic_read(&phba->fabric_iocb_count) == 0);
cmdiocb->iocb_cmpl = cmdiocb->fabric_iocb_cmpl;
cmdiocb->fabric_iocb_cmpl = NULL;
int ready;
int ret;
- if (atomic_read(&phba->fabric_iocb_count) > 1)
- BUG();
+ BUG_ON(atomic_read(&phba->fabric_iocb_count) > 1);
spin_lock_irqsave(&phba->hbalock, iflags);
ready = atomic_read(&phba->fabric_iocb_count) == 0 &&
LIST_HEAD(completions);
struct lpfc_hba *phba = ndlp->phba;
struct lpfc_iocbq *tmp_iocb, *piocb;
- struct lpfc_sli_ring *pring = &phba->sli.ring[LPFC_ELS_RING];
+ struct lpfc_sli_ring *pring;
+
+ pring = lpfc_phba_elsring(phba);
spin_lock_irq(&phba->hbalock);
list_for_each_entry_safe(piocb, tmp_iocb, &phba->fabric_iocb_list,
unsigned long iflag = 0;
spin_lock_irqsave(&phba->hbalock, iflag);
- spin_lock(&phba->sli4_hba.abts_sgl_list_lock);
+ spin_lock(&phba->sli4_hba.sgl_list_lock);
list_for_each_entry_safe(sglq_entry, sglq_next,
&phba->sli4_hba.lpfc_abts_els_sgl_list, list) {
if (sglq_entry->ndlp && sglq_entry->ndlp->vport == vport)
sglq_entry->ndlp = NULL;
}
- spin_unlock(&phba->sli4_hba.abts_sgl_list_lock);
+ spin_unlock(&phba->sli4_hba.sgl_list_lock);
spin_unlock_irqrestore(&phba->hbalock, iflag);
return;
}
struct lpfc_sglq *sglq_entry = NULL, *sglq_next = NULL;
unsigned long iflag = 0;
struct lpfc_nodelist *ndlp;
- struct lpfc_sli_ring *pring = &phba->sli.ring[LPFC_ELS_RING];
+ struct lpfc_sli_ring *pring;
+
+ pring = lpfc_phba_elsring(phba);
spin_lock_irqsave(&phba->hbalock, iflag);
- spin_lock(&phba->sli4_hba.abts_sgl_list_lock);
+ spin_lock(&phba->sli4_hba.sgl_list_lock);
list_for_each_entry_safe(sglq_entry, sglq_next,
&phba->sli4_hba.lpfc_abts_els_sgl_list, list) {
if (sglq_entry->sli4_xritag == xri) {
list_del(&sglq_entry->list);
ndlp = sglq_entry->ndlp;
sglq_entry->ndlp = NULL;
- spin_lock(&pring->ring_lock);
list_add_tail(&sglq_entry->list,
- &phba->sli4_hba.lpfc_sgl_list);
+ &phba->sli4_hba.lpfc_els_sgl_list);
sglq_entry->state = SGL_FREED;
- spin_unlock(&pring->ring_lock);
- spin_unlock(&phba->sli4_hba.abts_sgl_list_lock);
+ spin_unlock(&phba->sli4_hba.sgl_list_lock);
spin_unlock_irqrestore(&phba->hbalock, iflag);
lpfc_set_rrq_active(phba, ndlp,
sglq_entry->sli4_lxritag,
return;
}
}
- spin_unlock(&phba->sli4_hba.abts_sgl_list_lock);
+ spin_unlock(&phba->sli4_hba.sgl_list_lock);
lxri = lpfc_sli4_xri_inrange(phba, xri);
if (lxri == NO_XRI) {
spin_unlock_irqrestore(&phba->hbalock, iflag);
return;
}
- spin_lock(&pring->ring_lock);
+ spin_lock(&phba->sli4_hba.sgl_list_lock);
sglq_entry = __lpfc_get_active_sglq(phba, lxri);
if (!sglq_entry || (sglq_entry->sli4_xritag != xri)) {
- spin_unlock(&pring->ring_lock);
+ spin_unlock(&phba->sli4_hba.sgl_list_lock);
spin_unlock_irqrestore(&phba->hbalock, iflag);
return;
}
sglq_entry->state = SGL_XRI_ABORTED;
- spin_unlock(&pring->ring_lock);
+ spin_unlock(&phba->sli4_hba.sgl_list_lock);
spin_unlock_irqrestore(&phba->hbalock, iflag);
return;
}
/*******************************************************************
* This file is part of the Emulex Linux Device Driver for *
* Fibre Channel Host Bus Adapters. *
+ * Copyright (C) 2017 Broadcom. All Rights Reserved. The term *
+ * “Broadcom” refers to Broadcom Limited and/or its subsidiaries. *
* Copyright (C) 2004-2016 Emulex. All rights reserved. *
* EMULEX and SLI are trademarks of Emulex. *
- * www.emulex.com *
+ * www.broadcom.com *
* Portions Copyright (C) 2004-2005 Christoph Hellwig *
* *
* This program is free software; you can redistribute it and/or *
#include <scsi/scsi_device.h>
#include <scsi/scsi_host.h>
#include <scsi/scsi_transport_fc.h>
+#include <scsi/fc/fc_fs.h>
+
+#include <linux/nvme-fc-driver.h>
#include "lpfc_hw4.h"
#include "lpfc_hw.h"
#include "lpfc_disc.h"
#include "lpfc_sli.h"
#include "lpfc_sli4.h"
-#include "lpfc_scsi.h"
#include "lpfc.h"
+#include "lpfc_scsi.h"
+#include "lpfc_nvme.h"
#include "lpfc_logmsg.h"
#include "lpfc_crtn.h"
#include "lpfc_vport.h"
if (ndlp->nlp_sid != NLP_NO_SID) {
lpfc_sli_abort_iocb(ndlp->vport,
- &phba->sli.ring[phba->sli.fcp_ring],
+ &phba->sli.sli3_ring[LPFC_FCP_RING],
ndlp->nlp_sid, 0, LPFC_CTX_TGT);
}
}
if (ndlp->nlp_sid != NLP_NO_SID) {
/* flush the target */
lpfc_sli_abort_iocb(vport,
- &phba->sli.ring[phba->sli.fcp_ring],
- ndlp->nlp_sid, 0, LPFC_CTX_TGT);
+ &phba->sli.sli3_ring[LPFC_FCP_RING],
+ ndlp->nlp_sid, 0, LPFC_CTX_TGT);
}
put_node = rdata->pnode != NULL;
rdata->pnode = NULL;
if (ndlp->nlp_sid != NLP_NO_SID) {
warn_on = 1;
- lpfc_sli_abort_iocb(vport, &phba->sli.ring[phba->sli.fcp_ring],
+ lpfc_sli_abort_iocb(vport, &phba->sli.sli3_ring[LPFC_FCP_RING],
ndlp->nlp_sid, 0, LPFC_CTX_TGT);
}
return;
}
- fc_host_post_vendor_event(shost,
- fc_get_event_number(),
- evt_data_size,
- evt_data,
- LPFC_NL_VENDOR_ID);
+ if (phba->cfg_enable_fc4_type != LPFC_ENABLE_NVME)
+ fc_host_post_vendor_event(shost,
+ fc_get_event_number(),
+ evt_data_size,
+ evt_data,
+ LPFC_NL_VENDOR_ID);
lpfc_free_fast_evt(phba, fast_evt_data);
return;
}
lpfc_destroy_vport_work_array(phba, vports);
- pring = &phba->sli.ring[LPFC_ELS_RING];
+ pring = lpfc_phba_elsring(phba);
status = (ha_copy & (HA_RXMASK << (4*LPFC_ELS_RING)));
status >>= (4*LPFC_ELS_RING);
if ((status & HA_RXMASK) ||
void
lpfc_linkdown_port(struct lpfc_vport *vport)
{
+ struct lpfc_hba *phba = vport->phba;
struct Scsi_Host *shost = lpfc_shost_from_vport(vport);
- fc_host_post_event(shost, fc_get_event_number(), FCH_EVT_LINKDOWN, 0);
+ if (phba->cfg_enable_fc4_type != LPFC_ENABLE_NVME)
+ fc_host_post_event(shost, fc_get_event_number(),
+ FCH_EVT_LINKDOWN, 0);
lpfc_debugfs_disc_trc(vport, LPFC_DISC_TRC_ELS_CMD,
"Link Down: state:x%x rtry:x%x flg:x%x",
spin_unlock_irq(shost->host_lock);
}
vports = lpfc_create_vport_work_array(phba);
- if (vports != NULL)
+ if (vports != NULL) {
for (i = 0; i <= phba->max_vports && vports[i] != NULL; i++) {
/* Issue a LINK DOWN event to all nodes */
lpfc_linkdown_port(vports[i]);
+
+ vports[i]->fc_myDID = 0;
+
+ if ((phba->cfg_enable_fc4_type == LPFC_ENABLE_BOTH) ||
+ (phba->cfg_enable_fc4_type == LPFC_ENABLE_NVME)) {
+ if (phba->nvmet_support)
+ lpfc_nvmet_update_targetport(phba);
+ else
+ lpfc_nvme_update_localport(vports[i]);
+ }
}
+ }
lpfc_destroy_vport_work_array(phba, vports);
/* Clean up any firmware default rpi's */
mb = mempool_alloc(phba->mbox_mem_pool, GFP_KERNEL);
/* Setup myDID for link up if we are in pt2pt mode */
if (phba->pport->fc_flag & FC_PT2PT) {
- phba->pport->fc_myDID = 0;
mb = mempool_alloc(phba->mbox_mem_pool, GFP_KERNEL);
if (mb) {
lpfc_config_link(phba, mb);
phba->pport->fc_flag &= ~(FC_PT2PT | FC_PT2PT_PLOGI);
spin_unlock_irq(shost->host_lock);
}
-
return 0;
}
(vport != phba->pport))
return;
- fc_host_post_event(shost, fc_get_event_number(), FCH_EVT_LINKUP, 0);
+ if (phba->cfg_enable_fc4_type != LPFC_ENABLE_NVME)
+ fc_host_post_event(shost, fc_get_event_number(),
+ FCH_EVT_LINKUP, 0);
spin_lock_irq(shost->host_lock);
vport->fc_flag &= ~(FC_PT2PT | FC_PT2PT_PLOGI | FC_ABORT_DISCOVERY |
* This routine handles processing a CLEAR_LA mailbox
* command upon completion. It is setup in the LPFC_MBOXQ
* as the completion routine when the command is
- * handed off to the SLI layer.
+ * handed off to the SLI layer. SLI3 only.
*/
static void
lpfc_mbx_cmpl_clear_la(struct lpfc_hba *phba, LPFC_MBOXQ_t *pmb)
uint32_t control;
/* Since we don't do discovery right now, turn these off here */
- psli->ring[psli->extra_ring].flag &= ~LPFC_STOP_IOCB_EVENT;
- psli->ring[psli->fcp_ring].flag &= ~LPFC_STOP_IOCB_EVENT;
- psli->ring[psli->next_ring].flag &= ~LPFC_STOP_IOCB_EVENT;
+ psli->sli3_ring[LPFC_EXTRA_RING].flag &= ~LPFC_STOP_IOCB_EVENT;
+ psli->sli3_ring[LPFC_FCP_RING].flag &= ~LPFC_STOP_IOCB_EVENT;
/* Check for error */
if ((mb->mbxStatus) && (mb->mbxStatus != 0x1601)) {
* This routine handles processing a READ_TOPOLOGY mailbox
* command upon completion. It is setup in the LPFC_MBOXQ
* as the completion routine when the command is
- * handed off to the SLI layer.
+ * handed off to the SLI layer. SLI4 only.
*/
void
lpfc_mbx_cmpl_read_topology(struct lpfc_hba *phba, LPFC_MBOXQ_t *pmb)
struct lpfc_vport *vport = pmb->vport;
struct Scsi_Host *shost = lpfc_shost_from_vport(vport);
struct lpfc_mbx_read_top *la;
+ struct lpfc_sli_ring *pring;
MAILBOX_t *mb = &pmb->u.mb;
struct lpfc_dmabuf *mp = (struct lpfc_dmabuf *) (pmb->context1);
/* Unblock ELS traffic */
- phba->sli.ring[LPFC_ELS_RING].flag &= ~LPFC_STOP_IOCB_EVENT;
+ pring = lpfc_phba_elsring(phba);
+ pring->flag &= ~LPFC_STOP_IOCB_EVENT;
+
/* Check for error */
if (mb->mbxStatus) {
lpfc_printf_log(phba, KERN_INFO, LOG_LINK_EVENT,
spin_lock_irq(shost->host_lock);
ndlp->nlp_flag &= ~NLP_IGNR_REG_CMPL;
spin_unlock_irq(shost->host_lock);
+
+ /*
+ * We cannot leave the RPI registered because
+ * if we go thru discovery again for this ndlp
+ * a subsequent REG_RPI will fail.
+ */
+ ndlp->nlp_flag |= NLP_RPI_REGISTERED;
+ lpfc_unreg_rpi(vport, ndlp);
}
/* Call state machine */
vport->fc_flag &= ~(FC_FABRIC | FC_PUBLIC_LOOP);
spin_unlock_irq(shost->host_lock);
vport->fc_myDID = 0;
+
+ if ((phba->cfg_enable_fc4_type == LPFC_ENABLE_BOTH) ||
+ (phba->cfg_enable_fc4_type == LPFC_ENABLE_NVME)) {
+ if (phba->nvmet_support)
+ lpfc_nvmet_update_targetport(phba);
+ else
+ lpfc_nvme_update_localport(vport);
+ }
goto out;
}
return;
}
+ /*
+ * This routine will issue a GID_FT for each FC4 Type supported
+ * by the driver. ALL GID_FTs must complete before discovery is started.
+ */
+int
+lpfc_issue_gidft(struct lpfc_vport *vport)
+{
+ struct lpfc_hba *phba = vport->phba;
+
+ /* Good status, issue CT Request to NameServer */
+ if ((phba->cfg_enable_fc4_type == LPFC_ENABLE_BOTH) ||
+ (phba->cfg_enable_fc4_type == LPFC_ENABLE_FCP)) {
+ if (lpfc_ns_cmd(vport, SLI_CTNS_GID_FT, 0, SLI_CTPT_FCP)) {
+ /* Cannot issue NameServer FCP Query, so finish up
+ * discovery
+ */
+ lpfc_printf_vlog(vport, KERN_ERR, LOG_SLI,
+ "0604 %s FC TYPE %x %s\n",
+ "Failed to issue GID_FT to ",
+ FC_TYPE_FCP,
+ "Finishing discovery.");
+ return 0;
+ }
+ vport->gidft_inp++;
+ }
+
+ if ((phba->cfg_enable_fc4_type == LPFC_ENABLE_BOTH) ||
+ (phba->cfg_enable_fc4_type == LPFC_ENABLE_NVME)) {
+ if (lpfc_ns_cmd(vport, SLI_CTNS_GID_FT, 0, SLI_CTPT_NVME)) {
+ /* Cannot issue NameServer NVME Query, so finish up
+ * discovery
+ */
+ lpfc_printf_vlog(vport, KERN_ERR, LOG_SLI,
+ "0605 %s FC_TYPE %x %s %d\n",
+ "Failed to issue GID_FT to ",
+ FC_TYPE_NVME,
+ "Finishing discovery: gidftinp ",
+ vport->gidft_inp);
+ if (vport->gidft_inp == 0)
+ return 0;
+ } else
+ vport->gidft_inp++;
+ }
+ return vport->gidft_inp;
+}
+
/*
* This routine handles processing a NameServer REG_LOGIN mailbox
* command upon completion. It is setup in the LPFC_MBOXQ
pmb->context1 = NULL;
pmb->context2 = NULL;
+ vport->gidft_inp = 0;
if (mb->mbxStatus) {
-out:
lpfc_printf_vlog(vport, KERN_ERR, LOG_ELS,
"0260 Register NameServer error: 0x%x\n",
mb->mbxStatus);
+
+out:
/* decrement the node reference count held for this
* callback function.
*/
lpfc_ns_cmd(vport, SLI_CTNS_RSNN_NN, 0, 0);
lpfc_ns_cmd(vport, SLI_CTNS_RSPN_ID, 0, 0);
lpfc_ns_cmd(vport, SLI_CTNS_RFT_ID, 0, 0);
- lpfc_ns_cmd(vport, SLI_CTNS_RFF_ID, 0, 0);
+
+ if ((phba->cfg_enable_fc4_type == LPFC_ENABLE_BOTH) ||
+ (phba->cfg_enable_fc4_type == LPFC_ENABLE_FCP))
+ lpfc_ns_cmd(vport, SLI_CTNS_RFF_ID, 0, FC_TYPE_FCP);
+
+ if ((phba->cfg_enable_fc4_type == LPFC_ENABLE_BOTH) ||
+ (phba->cfg_enable_fc4_type == LPFC_ENABLE_NVME))
+ lpfc_ns_cmd(vport, SLI_CTNS_RFF_ID, 0,
+ FC_TYPE_NVME);
/* Issue SCR just before NameServer GID_FT Query */
lpfc_issue_els_scr(vport, SCR_DID, 0);
}
vport->fc_ns_retry = 0;
- /* Good status, issue CT Request to NameServer */
- if (lpfc_ns_cmd(vport, SLI_CTNS_GID_FT, 0, 0)) {
- /* Cannot issue NameServer Query, so finish up discovery */
+ if (lpfc_issue_gidft(vport) == 0)
goto out;
- }
- /* decrement the node reference count held for this
+ /*
+ * At this point in time we may need to wait for multiple
+ * SLI_CTNS_GID_FT CT commands to complete before we start discovery.
+ *
+ * decrement the node reference count held for this
* callback function.
*/
lpfc_nlp_put(ndlp);
struct fc_rport_identifiers rport_ids;
struct lpfc_hba *phba = vport->phba;
+ if (phba->cfg_enable_fc4_type == LPFC_ENABLE_NVME)
+ return;
+
/* Remote port has reappeared. Re-register w/ FC transport */
rport_ids.node_name = wwn_to_u64(ndlp->nlp_nodename.u.wwn);
rport_ids.port_name = wwn_to_u64(ndlp->nlp_portname.u.wwn);
lpfc_unregister_remote_port(struct lpfc_nodelist *ndlp)
{
struct fc_rport *rport = ndlp->rport;
+ struct lpfc_vport *vport = ndlp->vport;
+ struct lpfc_hba *phba = vport->phba;
- lpfc_debugfs_disc_trc(ndlp->vport, LPFC_DISC_TRC_RPORT,
+ if (phba->cfg_enable_fc4_type == LPFC_ENABLE_NVME)
+ return;
+
+ lpfc_debugfs_disc_trc(vport, LPFC_DISC_TRC_RPORT,
"rport delete: did:x%x flg:x%x type x%x",
ndlp->nlp_DID, ndlp->nlp_flag, ndlp->nlp_type);
- lpfc_printf_vlog(ndlp->vport, KERN_INFO, LOG_NODE,
+ lpfc_printf_vlog(vport, KERN_INFO, LOG_NODE,
"3184 rport unregister x%06x, rport %p\n",
ndlp->nlp_DID, rport);
int old_state, int new_state)
{
struct Scsi_Host *shost = lpfc_shost_from_vport(vport);
+ struct lpfc_hba *phba = vport->phba;
if (new_state == NLP_STE_UNMAPPED_NODE) {
ndlp->nlp_flag &= ~NLP_NODEV_REMOVE;
if (new_state == NLP_STE_NPR_NODE)
ndlp->nlp_flag &= ~NLP_RCV_PLOGI;
- /* Transport interface */
- if (ndlp->rport && (old_state == NLP_STE_MAPPED_NODE ||
- old_state == NLP_STE_UNMAPPED_NODE)) {
- vport->phba->nport_event_cnt++;
- lpfc_unregister_remote_port(ndlp);
+ /* FCP and NVME Transport interface */
+ if ((old_state == NLP_STE_MAPPED_NODE ||
+ old_state == NLP_STE_UNMAPPED_NODE)) {
+ if (ndlp->rport) {
+ vport->phba->nport_event_cnt++;
+ lpfc_unregister_remote_port(ndlp);
+ }
+
+ /* Notify the NVME transport of this rport's loss */
+ if (((phba->cfg_enable_fc4_type == LPFC_ENABLE_BOTH) ||
+ (phba->cfg_enable_fc4_type == LPFC_ENABLE_NVME)) &&
+ (vport->phba->nvmet_support == 0) &&
+ ((ndlp->nlp_fc4_type & NLP_FC4_NVME) ||
+ (ndlp->nlp_DID == Fabric_DID))) {
+ vport->phba->nport_event_cnt++;
+ lpfc_nvme_unregister_port(vport, ndlp);
+ }
}
+ /* FCP and NVME Transport interfaces */
+
if (new_state == NLP_STE_MAPPED_NODE ||
new_state == NLP_STE_UNMAPPED_NODE) {
- vport->phba->nport_event_cnt++;
- /*
- * Tell the fc transport about the port, if we haven't
- * already. If we have, and it's a scsi entity, be
- * sure to unblock any attached scsi devices
- */
- lpfc_register_remote_port(vport, ndlp);
+ if ((ndlp->nlp_fc4_type & NLP_FC4_FCP) ||
+ (ndlp->nlp_DID == Fabric_DID)) {
+ vport->phba->nport_event_cnt++;
+ /*
+ * Tell the fc transport about the port, if we haven't
+ * already. If we have, and it's a scsi entity, be
+ */
+ lpfc_register_remote_port(vport, ndlp);
+ }
+ /* Notify the NVME transport of this new rport. */
+ if (ndlp->nlp_fc4_type & NLP_FC4_NVME) {
+ if (vport->phba->nvmet_support == 0) {
+ /* Register this rport with the transport.
+ * Initiators take the NDLP ref count in
+ * the register.
+ */
+ vport->phba->nport_event_cnt++;
+ lpfc_nvme_register_port(vport, ndlp);
+ } else {
+ /* Just take an NDLP ref count since the
+ * target does not register rports.
+ */
+ lpfc_nlp_get(ndlp);
+ }
+ }
}
+
if ((new_state == NLP_STE_MAPPED_NODE) &&
(vport->stat_data_enabled)) {
/*
"0x%x\n", ndlp->nlp_DID);
}
/*
- * if we added to Mapped list, but the remote port
- * registration failed or assigned a target id outside
- * our presentable range - move the node to the
- * Unmapped List
+ * If the node just added to Mapped list was an FCP target,
+ * but the remote port registration failed or assigned a target
+ * id outside the presentable range - move the node to the
+ * Unmapped List.
*/
- if (new_state == NLP_STE_MAPPED_NODE &&
+ if ((new_state == NLP_STE_MAPPED_NODE) &&
+ (ndlp->nlp_type & NLP_FCP_TARGET) &&
(!ndlp->rport ||
ndlp->rport->scsi_target_id == -1 ||
ndlp->rport->scsi_target_id >= LPFC_MAX_TARGET)) {
ndlp->vport = vport;
ndlp->phba = vport->phba;
ndlp->nlp_sid = NLP_NO_SID;
+ ndlp->nlp_fc4_type = NLP_FC4_NONE;
kref_init(&ndlp->kref);
NLP_INT_NODE_ACT(ndlp);
atomic_set(&ndlp->cmd_pending, 0);
struct lpfc_iocbq *iocb,
struct lpfc_nodelist *ndlp)
{
- struct lpfc_sli *psli = &phba->sli;
IOCB_t *icmd = &iocb->iocb;
struct lpfc_vport *vport = ndlp->vport;
if (iocb->context1 == (uint8_t *) ndlp)
return 1;
}
- } else if (pring->ringno == psli->extra_ring) {
-
- } else if (pring->ringno == psli->fcp_ring) {
+ } else if (pring->ringno == LPFC_FCP_RING) {
/* Skip match check if waiting to relogin to FCP target */
if ((ndlp->nlp_type & NLP_FCP_TARGET) &&
(ndlp->nlp_flag & NLP_DELAY_TMO)) {
if (icmd->ulpContext == (volatile ushort)ndlp->nlp_rpi) {
return 1;
}
- } else if (pring->ringno == psli->next_ring) {
-
}
return 0;
}
+static void
+__lpfc_dequeue_nport_iocbs(struct lpfc_hba *phba,
+ struct lpfc_nodelist *ndlp, struct lpfc_sli_ring *pring,
+ struct list_head *dequeue_list)
+{
+ struct lpfc_iocbq *iocb, *next_iocb;
+
+ list_for_each_entry_safe(iocb, next_iocb, &pring->txq, list) {
+ /* Check to see if iocb matches the nport */
+ if (lpfc_check_sli_ndlp(phba, pring, iocb, ndlp))
+ /* match, dequeue */
+ list_move_tail(&iocb->list, dequeue_list);
+ }
+}
+
+static void
+lpfc_sli3_dequeue_nport_iocbs(struct lpfc_hba *phba,
+ struct lpfc_nodelist *ndlp, struct list_head *dequeue_list)
+{
+ struct lpfc_sli *psli = &phba->sli;
+ uint32_t i;
+
+ spin_lock_irq(&phba->hbalock);
+ for (i = 0; i < psli->num_rings; i++)
+ __lpfc_dequeue_nport_iocbs(phba, ndlp, &psli->sli3_ring[i],
+ dequeue_list);
+ spin_unlock_irq(&phba->hbalock);
+}
+
+static void
+lpfc_sli4_dequeue_nport_iocbs(struct lpfc_hba *phba,
+ struct lpfc_nodelist *ndlp, struct list_head *dequeue_list)
+{
+ struct lpfc_sli_ring *pring;
+ struct lpfc_queue *qp = NULL;
+
+ spin_lock_irq(&phba->hbalock);
+ list_for_each_entry(qp, &phba->sli4_hba.lpfc_wq_list, wq_list) {
+ pring = qp->pring;
+ if (!pring)
+ continue;
+ spin_lock_irq(&pring->ring_lock);
+ __lpfc_dequeue_nport_iocbs(phba, ndlp, pring, dequeue_list);
+ spin_unlock_irq(&pring->ring_lock);
+ }
+ spin_unlock_irq(&phba->hbalock);
+}
+
/*
* Free resources / clean up outstanding I/Os
* associated with nlp_rpi in the LPFC_NODELIST entry.
lpfc_no_rpi(struct lpfc_hba *phba, struct lpfc_nodelist *ndlp)
{
LIST_HEAD(completions);
- struct lpfc_sli *psli;
- struct lpfc_sli_ring *pring;
- struct lpfc_iocbq *iocb, *next_iocb;
- uint32_t i;
lpfc_fabric_abort_nport(ndlp);
* Everything that matches on txcmplq will be returned
* by firmware with a no rpi error.
*/
- psli = &phba->sli;
if (ndlp->nlp_flag & NLP_RPI_REGISTERED) {
- /* Now process each ring */
- for (i = 0; i < psli->num_rings; i++) {
- pring = &psli->ring[i];
-
- spin_lock_irq(&phba->hbalock);
- list_for_each_entry_safe(iocb, next_iocb, &pring->txq,
- list) {
- /*
- * Check to see if iocb matches the nport we are
- * looking for
- */
- if ((lpfc_check_sli_ndlp(phba, pring, iocb,
- ndlp))) {
- /* It matches, so deque and call compl
- with an error */
- list_move_tail(&iocb->list,
- &completions);
- }
- }
- spin_unlock_irq(&phba->hbalock);
- }
+ if (phba->sli_rev != LPFC_SLI_REV4)
+ lpfc_sli3_dequeue_nport_iocbs(phba, ndlp, &completions);
+ else
+ lpfc_sli4_dequeue_nport_iocbs(phba, ndlp, &completions);
}
/* Cancel all the IOCBs from the completions list */
return NULL;
lpfc_nlp_init(vport, ndlp, did);
lpfc_nlp_set_state(vport, ndlp, NLP_STE_NPR_NODE);
+ if (vport->phba->nvmet_support)
+ return ndlp;
spin_lock_irq(shost->host_lock);
ndlp->nlp_flag |= NLP_NPR_2B_DISC;
spin_unlock_irq(shost->host_lock);
ndlp = lpfc_enable_node(vport, ndlp, NLP_STE_NPR_NODE);
if (!ndlp)
return NULL;
+ if (vport->phba->nvmet_support)
+ return ndlp;
spin_lock_irq(shost->host_lock);
ndlp->nlp_flag |= NLP_NPR_2B_DISC;
spin_unlock_irq(shost->host_lock);
* delay timeout is not needed.
*/
lpfc_cancel_retry_delay_tmo(vport, ndlp);
+ if (vport->phba->nvmet_support)
+ return ndlp;
spin_lock_irq(shost->host_lock);
ndlp->nlp_flag |= NLP_NPR_2B_DISC;
spin_unlock_irq(shost->host_lock);
ndlp->nlp_flag & NLP_RCV_PLOGI)
return NULL;
lpfc_nlp_set_state(vport, ndlp, NLP_STE_NPR_NODE);
+ if (vport->phba->nvmet_support)
+ return ndlp;
spin_lock_irq(shost->host_lock);
ndlp->nlp_flag |= NLP_NPR_2B_DISC;
spin_unlock_irq(shost->host_lock);
return;
}
+/* SLI3 only */
void
lpfc_issue_clear_la(struct lpfc_hba *phba, struct lpfc_vport *vport)
{
LPFC_MBOXQ_t *mbox;
struct lpfc_sli *psli = &phba->sli;
- struct lpfc_sli_ring *extra_ring = &psli->ring[psli->extra_ring];
- struct lpfc_sli_ring *fcp_ring = &psli->ring[psli->fcp_ring];
- struct lpfc_sli_ring *next_ring = &psli->ring[psli->next_ring];
+ struct lpfc_sli_ring *extra_ring = &psli->sli3_ring[LPFC_EXTRA_RING];
+ struct lpfc_sli_ring *fcp_ring = &psli->sli3_ring[LPFC_FCP_RING];
int rc;
/*
lpfc_disc_flush_list(vport);
extra_ring->flag &= ~LPFC_STOP_IOCB_EVENT;
fcp_ring->flag &= ~LPFC_STOP_IOCB_EVENT;
- next_ring->flag &= ~LPFC_STOP_IOCB_EVENT;
phba->link_state = LPFC_HBA_ERROR;
}
}
struct lpfc_sli_ring *pring;
psli = &phba->sli;
- pring = &psli->ring[LPFC_ELS_RING];
+ pring = lpfc_phba_elsring(phba);
/* Error matching iocb on txq or txcmplq
* First check the txq.
switch (vport->port_state) {
case LPFC_LOCAL_CFG_LINK:
- /* port_state is identically LPFC_LOCAL_CFG_LINK while waiting for
- * FAN
- */
- /* FAN timeout */
+ /*
+ * port_state is identically LPFC_LOCAL_CFG_LINK while
+ * waiting for FAN timeout
+ */
lpfc_printf_vlog(vport, KERN_WARNING, LOG_DISCOVERY,
"0221 FAN timeout\n");
+
/* Start discovery by sending FLOGI, clean up old rpis */
list_for_each_entry_safe(ndlp, next_ndlp, &vport->fc_nodes,
nlp_listp) {
if (vport->fc_ns_retry < LPFC_MAX_NS_RETRY) {
/* Try it one more time */
vport->fc_ns_retry++;
- rc = lpfc_ns_cmd(vport, SLI_CTNS_GID_FT,
- vport->fc_ns_retry, 0);
+ vport->gidft_inp = 0;
+ rc = lpfc_issue_gidft(vport);
if (rc == 0)
break;
}
if (clrlaerr) {
lpfc_disc_flush_list(vport);
- psli->ring[(psli->extra_ring)].flag &= ~LPFC_STOP_IOCB_EVENT;
- psli->ring[(psli->fcp_ring)].flag &= ~LPFC_STOP_IOCB_EVENT;
- psli->ring[(psli->next_ring)].flag &= ~LPFC_STOP_IOCB_EVENT;
+ if (phba->sli_rev != LPFC_SLI_REV4) {
+ psli->sli3_ring[(LPFC_EXTRA_RING)].flag &=
+ ~LPFC_STOP_IOCB_EVENT;
+ psli->sli3_ring[LPFC_FCP_RING].flag &=
+ ~LPFC_STOP_IOCB_EVENT;
+ }
vport->port_state = LPFC_VPORT_READY;
}
-
return;
}
/*******************************************************************
* This file is part of the Emulex Linux Device Driver for *
* Fibre Channel Host Bus Adapters. *
+ * Copyright (C) 2017 Broadcom. All Rights Reserved. The term *
+ * “Broadcom” refers to Broadcom Limited and/or its subsidiaries. *
* Copyright (C) 2004-2016 Emulex. All rights reserved. *
* EMULEX and SLI are trademarks of Emulex. *
- * www.emulex.com *
+ * www.broadcom.com *
* *
* This program is free software; you can redistribute it and/or *
* modify it under the terms of version 2 of the GNU General *
#define LPFC_FCP_RING 0 /* ring 0 for FCP initiator commands */
#define LPFC_EXTRA_RING 1 /* ring 1 for other protocols */
#define LPFC_ELS_RING 2 /* ring 2 for ELS commands */
-#define LPFC_FCP_NEXT_RING 3
-#define LPFC_FCP_OAS_RING 3
#define SLI2_IOCB_CMD_R0_ENTRIES 172 /* SLI-2 FCP command ring entries */
#define SLI2_IOCB_RSP_R0_ENTRIES 134 /* SLI-2 FCP response ring entries */
uint32_t word;
};
-#define FC4_FEATURE_INIT 0x2
-#define FC4_FEATURE_TARGET 0x1
+/* FC4 Feature bits for RFF_ID */
+#define FC4_FEATURE_TARGET 0x1
+#define FC4_FEATURE_INIT 0x2
+#define FC4_FEATURE_NVME_DISC 0x4
struct lpfc_sli_ct_request {
/* Structure is in Big Endian format */
uint8_t AreaScope;
uint8_t Fc4Type; /* for GID_FT requests */
} gid;
+ struct gid_ff {
+ uint8_t Flags;
+ uint8_t DomainScope;
+ uint8_t AreaScope;
+ uint8_t rsvd1;
+ uint8_t rsvd2;
+ uint8_t rsvd3;
+ uint8_t Fc4FBits;
+ uint8_t Fc4Type;
+ } gid_ff;
struct rft {
uint32_t PortId; /* For RFT_ID requests */
struct gff_acc {
uint8_t fbits[128];
} gff_acc;
+ struct gft {
+ uint32_t PortId;
+ } gft;
+ struct gft_acc {
+ uint32_t fc4_types[8];
+ } gft_acc;
#define FCP_TYPE_FEATURE_OFFSET 7
struct rff {
uint32_t PortId;
#define SLI_CT_REVISION 1
#define GID_REQUEST_SZ (offsetof(struct lpfc_sli_ct_request, un) + \
sizeof(struct gid))
+#define GIDFF_REQUEST_SZ (offsetof(struct lpfc_sli_ct_request, un) + \
+ sizeof(struct gid_ff))
#define GFF_REQUEST_SZ (offsetof(struct lpfc_sli_ct_request, un) + \
sizeof(struct gff))
+#define GFT_REQUEST_SZ (offsetof(struct lpfc_sli_ct_request, un) + \
+ sizeof(struct gft))
#define RFT_REQUEST_SZ (offsetof(struct lpfc_sli_ct_request, un) + \
sizeof(struct rft))
#define RFF_REQUEST_SZ (offsetof(struct lpfc_sli_ct_request, un) + \
#define SLI_CTNS_GNN_IP 0x0153
#define SLI_CTNS_GIPA_IP 0x0156
#define SLI_CTNS_GID_FT 0x0171
+#define SLI_CTNS_GID_FF 0x01F1
#define SLI_CTNS_GID_PT 0x01A1
#define SLI_CTNS_RPN_ID 0x0212
#define SLI_CTNS_RNN_ID 0x0213
* Port Types
*/
-#define SLI_CTPT_N_PORT 0x01
-#define SLI_CTPT_NL_PORT 0x02
-#define SLI_CTPT_FNL_PORT 0x03
-#define SLI_CTPT_IP 0x04
-#define SLI_CTPT_FCP 0x08
-#define SLI_CTPT_NX_PORT 0x7F
-#define SLI_CTPT_F_PORT 0x81
-#define SLI_CTPT_FL_PORT 0x82
-#define SLI_CTPT_E_PORT 0x84
+#define SLI_CTPT_N_PORT 0x01
+#define SLI_CTPT_NL_PORT 0x02
+#define SLI_CTPT_FNL_PORT 0x03
+#define SLI_CTPT_IP 0x04
+#define SLI_CTPT_FCP 0x08
+#define SLI_CTPT_NVME 0x28
+#define SLI_CTPT_NX_PORT 0x7F
+#define SLI_CTPT_F_PORT 0x81
+#define SLI_CTPT_FL_PORT 0x82
+#define SLI_CTPT_E_PORT 0x84
#define SLI_CT_LAST_ENTRY 0x80000000
uint8_t IEEE[6]; /* FC IEEE address */
} s;
uint8_t wwn[8];
+ uint64_t name;
} u;
};
struct class_parms cls2;
struct class_parms cls3;
struct class_parms cls4;
- uint8_t vendorVersion[16];
+ union {
+ uint8_t vendorVersion[16];
+ struct {
+ uint32_t vid;
+#define LPFC_VV_EMLX_ID 0x454d4c58 /* EMLX */
+ uint32_t flags;
+#define LPFC_VV_SUPPRESS_RSP 1
+ } vv;
+ } un;
};
/*
#define ELS_CMD_REC 0x13000000
#define ELS_CMD_RDP 0x18000000
#define ELS_CMD_PRLI 0x20100014
+#define ELS_CMD_NVMEPRLI 0x20140018
#define ELS_CMD_PRLO 0x21100014
#define ELS_CMD_PRLO_ACC 0x02100014
#define ELS_CMD_PDISC 0x50000000
#define ELS_CMD_REC 0x13
#define ELS_CMD_RDP 0x18
#define ELS_CMD_PRLI 0x14001020
+#define ELS_CMD_NVMEPRLI 0x18001420
#define ELS_CMD_PRLO 0x14001021
#define ELS_CMD_PRLO_ACC 0x14001002
#define ELS_CMD_PDISC 0x50
uint8_t prliType; /* FC Parm Word 0, bit 24:31 */
#define PRLI_FCP_TYPE 0x08
+#define PRLI_NVME_TYPE 0x28
uint8_t word0Reserved1; /* FC Parm Word 0, bit 16:23 */
#ifdef __BIG_ENDIAN_BITFIELD
uint8_t vendor_name[16];
uint8_t model_number[16];
uint8_t serial_number[16];
- uint8_t revision[2];
- uint8_t reserved[2];
+ uint8_t revision[4];
uint8_t date[8];
};
struct fc_rdp_res_frame {
- uint32_t reply_sequence; /* FC word0 LS_ACC or LS_RJT */
- uint32_t length; /* FC Word 1 */
- struct fc_rdp_link_service_desc link_service_desc; /* Word 2 -4 */
- struct fc_rdp_sfp_desc sfp_desc; /* Word 5 -9 */
- struct fc_rdp_port_speed_desc portspeed_desc; /* Word 10-12 */
- struct fc_rdp_link_error_status_desc link_error_desc; /* Word 13-21 */
- struct fc_rdp_port_name_desc diag_port_names_desc; /* Word 22-27 */
- struct fc_rdp_port_name_desc attached_port_names_desc;/* Word 28-33 */
+ uint32_t reply_sequence; /* FC word0 LS_ACC or LS_RJT */
+ uint32_t length; /* FC Word 1 */
+ struct fc_rdp_link_service_desc link_service_desc; /* Word 2 -4 */
+ struct fc_rdp_sfp_desc sfp_desc; /* Word 5 -9 */
+ struct fc_rdp_port_speed_desc portspeed_desc; /* Word 10 -12 */
+ struct fc_rdp_link_error_status_desc link_error_desc; /* Word 13 -21 */
+ struct fc_rdp_port_name_desc diag_port_names_desc; /* Word 22 -27 */
+ struct fc_rdp_port_name_desc attached_port_names_desc;/* Word 28 -33 */
struct fc_fec_rdp_desc fec_desc; /* FC word 34-37*/
struct fc_rdp_bbc_desc bbc_desc; /* FC Word 38-42*/
struct fc_rdp_oed_sfp_desc oed_temp_desc; /* FC Word 43-47*/
#define MBX_INIT_VFI 0xA3
#define MBX_INIT_VPI 0xA4
#define MBX_ACCESS_VDATA 0xA5
+#define MBX_REG_FCFI_MRQ 0xAF
#define MBX_AUTH_PORT 0xF8
#define MBX_SECURITY_MGMT 0xF9
/*******************************************************************
* This file is part of the Emulex Linux Device Driver for *
* Fibre Channel Host Bus Adapters. *
- * Copyright (C) 2009-2016 Emulex. All rights reserved. *
+ * Copyright (C) 2017 Broadcom. All Rights Reserved. The term *
+ * “Broadcom” refers to Broadcom Limited and/or its subsidiaries. *
+ * Copyright (C) 2009-2016 Emulex. All rights reserved. *
* EMULEX and SLI are trademarks of Emulex. *
- * www.emulex.com *
+ * www.broadcom.com *
* *
* This program is free software; you can redistribute it and/or *
* modify it under the terms of version 2 of the GNU General *
#define LPFC_MAX_MQ_PAGE 8
#define LPFC_MAX_WQ_PAGE_V0 4
#define LPFC_MAX_WQ_PAGE 8
+#define LPFC_MAX_RQ_PAGE 8
#define LPFC_MAX_CQ_PAGE 4
#define LPFC_MAX_EQ_PAGE 8
/* Configuration of Interrupts / sec for entire HBA port */
#define LPFC_MIN_IMAX 5000
#define LPFC_MAX_IMAX 5000000
-#define LPFC_DEF_IMAX 50000
+#define LPFC_DEF_IMAX 150000
#define LPFC_MIN_CPU_MAP 0
#define LPFC_MAX_CPU_MAP 2
#define CQE_CODE_RECEIVE 0x4
#define CQE_CODE_XRI_ABORTED 0x5
#define CQE_CODE_RECEIVE_V1 0x9
+#define CQE_CODE_NVME_ERSP 0xd
/*
* Define mask value for xri_aborted and wcqe completed CQE extended status.
#define lpfc_wcqe_c_hw_status_SHIFT 0
#define lpfc_wcqe_c_hw_status_MASK 0x000000FF
#define lpfc_wcqe_c_hw_status_WORD word0
+#define lpfc_wcqe_c_ersp0_SHIFT 0
+#define lpfc_wcqe_c_ersp0_MASK 0x0000FFFF
+#define lpfc_wcqe_c_ersp0_WORD word0
uint32_t total_data_placed;
uint32_t parameter;
#define lpfc_wcqe_c_bg_edir_SHIFT 5
#define lpfc_wcqe_c_code_SHIFT lpfc_cqe_code_SHIFT
#define lpfc_wcqe_c_code_MASK lpfc_cqe_code_MASK
#define lpfc_wcqe_c_code_WORD lpfc_cqe_code_WORD
+#define lpfc_wcqe_c_sqhead_SHIFT 0
+#define lpfc_wcqe_c_sqhead_MASK 0x0000FFFF
+#define lpfc_wcqe_c_sqhead_WORD word3
};
/* completion queue entry for wqe release */
#define LPFC_MBOX_OPCODE_FCOE_DELETE_FCF 0x0A
#define LPFC_MBOX_OPCODE_FCOE_POST_HDR_TEMPLATE 0x0B
#define LPFC_MBOX_OPCODE_FCOE_REDISCOVER_FCF 0x10
+#define LPFC_MBOX_OPCODE_FCOE_CQ_CREATE_SET 0x1D
#define LPFC_MBOX_OPCODE_FCOE_SET_FCLINK_SETTINGS 0x21
#define LPFC_MBOX_OPCODE_FCOE_LINK_DIAG_STATE 0x22
#define LPFC_MBOX_OPCODE_FCOE_LINK_DIAG_LOOPBACK 0x23
} u;
};
+struct lpfc_mbx_cq_create_set {
+ union lpfc_sli4_cfg_shdr cfg_shdr;
+ union {
+ struct {
+ uint32_t word0;
+#define lpfc_mbx_cq_create_set_page_size_SHIFT 16 /* Version 2 Only */
+#define lpfc_mbx_cq_create_set_page_size_MASK 0x000000FF
+#define lpfc_mbx_cq_create_set_page_size_WORD word0
+#define lpfc_mbx_cq_create_set_num_pages_SHIFT 0
+#define lpfc_mbx_cq_create_set_num_pages_MASK 0x0000FFFF
+#define lpfc_mbx_cq_create_set_num_pages_WORD word0
+ uint32_t word1;
+#define lpfc_mbx_cq_create_set_evt_SHIFT 31
+#define lpfc_mbx_cq_create_set_evt_MASK 0x00000001
+#define lpfc_mbx_cq_create_set_evt_WORD word1
+#define lpfc_mbx_cq_create_set_valid_SHIFT 29
+#define lpfc_mbx_cq_create_set_valid_MASK 0x00000001
+#define lpfc_mbx_cq_create_set_valid_WORD word1
+#define lpfc_mbx_cq_create_set_cqe_cnt_SHIFT 27
+#define lpfc_mbx_cq_create_set_cqe_cnt_MASK 0x00000003
+#define lpfc_mbx_cq_create_set_cqe_cnt_WORD word1
+#define lpfc_mbx_cq_create_set_cqe_size_SHIFT 25
+#define lpfc_mbx_cq_create_set_cqe_size_MASK 0x00000003
+#define lpfc_mbx_cq_create_set_cqe_size_WORD word1
+#define lpfc_mbx_cq_create_set_auto_SHIFT 15
+#define lpfc_mbx_cq_create_set_auto_MASK 0x0000001
+#define lpfc_mbx_cq_create_set_auto_WORD word1
+#define lpfc_mbx_cq_create_set_nodelay_SHIFT 14
+#define lpfc_mbx_cq_create_set_nodelay_MASK 0x00000001
+#define lpfc_mbx_cq_create_set_nodelay_WORD word1
+#define lpfc_mbx_cq_create_set_clswm_SHIFT 12
+#define lpfc_mbx_cq_create_set_clswm_MASK 0x00000003
+#define lpfc_mbx_cq_create_set_clswm_WORD word1
+ uint32_t word2;
+#define lpfc_mbx_cq_create_set_arm_SHIFT 31
+#define lpfc_mbx_cq_create_set_arm_MASK 0x00000001
+#define lpfc_mbx_cq_create_set_arm_WORD word2
+#define lpfc_mbx_cq_create_set_num_cq_SHIFT 0
+#define lpfc_mbx_cq_create_set_num_cq_MASK 0x0000FFFF
+#define lpfc_mbx_cq_create_set_num_cq_WORD word2
+ uint32_t word3;
+#define lpfc_mbx_cq_create_set_eq_id1_SHIFT 16
+#define lpfc_mbx_cq_create_set_eq_id1_MASK 0x0000FFFF
+#define lpfc_mbx_cq_create_set_eq_id1_WORD word3
+#define lpfc_mbx_cq_create_set_eq_id0_SHIFT 0
+#define lpfc_mbx_cq_create_set_eq_id0_MASK 0x0000FFFF
+#define lpfc_mbx_cq_create_set_eq_id0_WORD word3
+ uint32_t word4;
+#define lpfc_mbx_cq_create_set_eq_id3_SHIFT 16
+#define lpfc_mbx_cq_create_set_eq_id3_MASK 0x0000FFFF
+#define lpfc_mbx_cq_create_set_eq_id3_WORD word4
+#define lpfc_mbx_cq_create_set_eq_id2_SHIFT 0
+#define lpfc_mbx_cq_create_set_eq_id2_MASK 0x0000FFFF
+#define lpfc_mbx_cq_create_set_eq_id2_WORD word4
+ uint32_t word5;
+#define lpfc_mbx_cq_create_set_eq_id5_SHIFT 16
+#define lpfc_mbx_cq_create_set_eq_id5_MASK 0x0000FFFF
+#define lpfc_mbx_cq_create_set_eq_id5_WORD word5
+#define lpfc_mbx_cq_create_set_eq_id4_SHIFT 0
+#define lpfc_mbx_cq_create_set_eq_id4_MASK 0x0000FFFF
+#define lpfc_mbx_cq_create_set_eq_id4_WORD word5
+ uint32_t word6;
+#define lpfc_mbx_cq_create_set_eq_id7_SHIFT 16
+#define lpfc_mbx_cq_create_set_eq_id7_MASK 0x0000FFFF
+#define lpfc_mbx_cq_create_set_eq_id7_WORD word6
+#define lpfc_mbx_cq_create_set_eq_id6_SHIFT 0
+#define lpfc_mbx_cq_create_set_eq_id6_MASK 0x0000FFFF
+#define lpfc_mbx_cq_create_set_eq_id6_WORD word6
+ uint32_t word7;
+#define lpfc_mbx_cq_create_set_eq_id9_SHIFT 16
+#define lpfc_mbx_cq_create_set_eq_id9_MASK 0x0000FFFF
+#define lpfc_mbx_cq_create_set_eq_id9_WORD word7
+#define lpfc_mbx_cq_create_set_eq_id8_SHIFT 0
+#define lpfc_mbx_cq_create_set_eq_id8_MASK 0x0000FFFF
+#define lpfc_mbx_cq_create_set_eq_id8_WORD word7
+ uint32_t word8;
+#define lpfc_mbx_cq_create_set_eq_id11_SHIFT 16
+#define lpfc_mbx_cq_create_set_eq_id11_MASK 0x0000FFFF
+#define lpfc_mbx_cq_create_set_eq_id11_WORD word8
+#define lpfc_mbx_cq_create_set_eq_id10_SHIFT 0
+#define lpfc_mbx_cq_create_set_eq_id10_MASK 0x0000FFFF
+#define lpfc_mbx_cq_create_set_eq_id10_WORD word8
+ uint32_t word9;
+#define lpfc_mbx_cq_create_set_eq_id13_SHIFT 16
+#define lpfc_mbx_cq_create_set_eq_id13_MASK 0x0000FFFF
+#define lpfc_mbx_cq_create_set_eq_id13_WORD word9
+#define lpfc_mbx_cq_create_set_eq_id12_SHIFT 0
+#define lpfc_mbx_cq_create_set_eq_id12_MASK 0x0000FFFF
+#define lpfc_mbx_cq_create_set_eq_id12_WORD word9
+ uint32_t word10;
+#define lpfc_mbx_cq_create_set_eq_id15_SHIFT 16
+#define lpfc_mbx_cq_create_set_eq_id15_MASK 0x0000FFFF
+#define lpfc_mbx_cq_create_set_eq_id15_WORD word10
+#define lpfc_mbx_cq_create_set_eq_id14_SHIFT 0
+#define lpfc_mbx_cq_create_set_eq_id14_MASK 0x0000FFFF
+#define lpfc_mbx_cq_create_set_eq_id14_WORD word10
+ struct dma_address page[1];
+ } request;
+ struct {
+ uint32_t word0;
+#define lpfc_mbx_cq_create_set_num_alloc_SHIFT 16
+#define lpfc_mbx_cq_create_set_num_alloc_MASK 0x0000FFFF
+#define lpfc_mbx_cq_create_set_num_alloc_WORD word0
+#define lpfc_mbx_cq_create_set_base_id_SHIFT 0
+#define lpfc_mbx_cq_create_set_base_id_MASK 0x0000FFFF
+#define lpfc_mbx_cq_create_set_base_id_WORD word0
+ } response;
+ } u;
+};
+
struct lpfc_mbx_cq_destroy {
struct mbox_header header;
union {
#define lpfc_mbx_wq_create_page_size_SHIFT 0
#define lpfc_mbx_wq_create_page_size_MASK 0x000000FF
#define lpfc_mbx_wq_create_page_size_WORD word1
+#define LPFC_WQ_PAGE_SIZE_4096 0x1
#define lpfc_mbx_wq_create_wqe_size_SHIFT 8
#define lpfc_mbx_wq_create_wqe_size_MASK 0x0000000F
#define lpfc_mbx_wq_create_wqe_size_WORD word1
#define LPFC_RQ_RING_SIZE_1024 10 /* 1024 entries */
#define LPFC_RQ_RING_SIZE_2048 11 /* 2048 entries */
#define LPFC_RQ_RING_SIZE_4096 12 /* 4096 entries */
-#define lpfc_rq_context_rqe_count_1_SHIFT 16 /* Version 1 Only */
+#define lpfc_rq_context_rqe_count_1_SHIFT 16 /* Version 1-2 Only */
#define lpfc_rq_context_rqe_count_1_MASK 0x0000FFFF
#define lpfc_rq_context_rqe_count_1_WORD word0
-#define lpfc_rq_context_rqe_size_SHIFT 8 /* Version 1 Only */
+#define lpfc_rq_context_rqe_size_SHIFT 8 /* Version 1-2 Only */
#define lpfc_rq_context_rqe_size_MASK 0x0000000F
#define lpfc_rq_context_rqe_size_WORD word0
#define LPFC_RQE_SIZE_8 2
#define lpfc_rq_context_page_size_SHIFT 0 /* Version 1 Only */
#define lpfc_rq_context_page_size_MASK 0x000000FF
#define lpfc_rq_context_page_size_WORD word0
- uint32_t reserved1;
+#define LPFC_RQ_PAGE_SIZE_4096 0x1
+ uint32_t word1;
+#define lpfc_rq_context_data_size_SHIFT 16 /* Version 2 Only */
+#define lpfc_rq_context_data_size_MASK 0x0000FFFF
+#define lpfc_rq_context_data_size_WORD word1
+#define lpfc_rq_context_hdr_size_SHIFT 0 /* Version 2 Only */
+#define lpfc_rq_context_hdr_size_MASK 0x0000FFFF
+#define lpfc_rq_context_hdr_size_WORD word1
uint32_t word2;
#define lpfc_rq_context_cq_id_SHIFT 16
#define lpfc_rq_context_cq_id_MASK 0x000003FF
#define lpfc_rq_context_buf_size_SHIFT 0
#define lpfc_rq_context_buf_size_MASK 0x0000FFFF
#define lpfc_rq_context_buf_size_WORD word2
+#define lpfc_rq_context_base_cq_SHIFT 0 /* Version 2 Only */
+#define lpfc_rq_context_base_cq_MASK 0x0000FFFF
+#define lpfc_rq_context_base_cq_WORD word2
uint32_t buffer_size; /* Version 1 Only */
};
#define lpfc_mbx_rq_create_ulp_num_MASK 0x000000FF
#define lpfc_mbx_rq_create_ulp_num_WORD word0
struct rq_context context;
- struct dma_address page[LPFC_MAX_WQ_PAGE];
+ struct dma_address page[LPFC_MAX_RQ_PAGE];
} request;
struct {
uint32_t word0;
+#define lpfc_mbx_rq_create_q_cnt_v2_SHIFT 16
+#define lpfc_mbx_rq_create_q_cnt_v2_MASK 0x0000FFFF
+#define lpfc_mbx_rq_create_q_cnt_v2_WORD word0
+#define lpfc_mbx_rq_create_q_id_SHIFT 0
+#define lpfc_mbx_rq_create_q_id_MASK 0x0000FFFF
+#define lpfc_mbx_rq_create_q_id_WORD word0
+ uint32_t doorbell_offset;
+ uint32_t word2;
+#define lpfc_mbx_rq_create_bar_set_SHIFT 0
+#define lpfc_mbx_rq_create_bar_set_MASK 0x0000FFFF
+#define lpfc_mbx_rq_create_bar_set_WORD word2
+#define lpfc_mbx_rq_create_db_format_SHIFT 16
+#define lpfc_mbx_rq_create_db_format_MASK 0x0000FFFF
+#define lpfc_mbx_rq_create_db_format_WORD word2
+ } response;
+ } u;
+};
+
+struct lpfc_mbx_rq_create_v2 {
+ union lpfc_sli4_cfg_shdr cfg_shdr;
+ union {
+ struct {
+ uint32_t word0;
+#define lpfc_mbx_rq_create_num_pages_SHIFT 0
+#define lpfc_mbx_rq_create_num_pages_MASK 0x0000FFFF
+#define lpfc_mbx_rq_create_num_pages_WORD word0
+#define lpfc_mbx_rq_create_rq_cnt_SHIFT 16
+#define lpfc_mbx_rq_create_rq_cnt_MASK 0x000000FF
+#define lpfc_mbx_rq_create_rq_cnt_WORD word0
+#define lpfc_mbx_rq_create_dua_SHIFT 16
+#define lpfc_mbx_rq_create_dua_MASK 0x00000001
+#define lpfc_mbx_rq_create_dua_WORD word0
+#define lpfc_mbx_rq_create_bqu_SHIFT 17
+#define lpfc_mbx_rq_create_bqu_MASK 0x00000001
+#define lpfc_mbx_rq_create_bqu_WORD word0
+#define lpfc_mbx_rq_create_ulp_num_SHIFT 24
+#define lpfc_mbx_rq_create_ulp_num_MASK 0x000000FF
+#define lpfc_mbx_rq_create_ulp_num_WORD word0
+#define lpfc_mbx_rq_create_dim_SHIFT 29
+#define lpfc_mbx_rq_create_dim_MASK 0x00000001
+#define lpfc_mbx_rq_create_dim_WORD word0
+#define lpfc_mbx_rq_create_dfd_SHIFT 30
+#define lpfc_mbx_rq_create_dfd_MASK 0x00000001
+#define lpfc_mbx_rq_create_dfd_WORD word0
+#define lpfc_mbx_rq_create_dnb_SHIFT 31
+#define lpfc_mbx_rq_create_dnb_MASK 0x00000001
+#define lpfc_mbx_rq_create_dnb_WORD word0
+ struct rq_context context;
+ struct dma_address page[1];
+ } request;
+ struct {
+ uint32_t word0;
+#define lpfc_mbx_rq_create_q_cnt_v2_SHIFT 16
+#define lpfc_mbx_rq_create_q_cnt_v2_MASK 0x0000FFFF
+#define lpfc_mbx_rq_create_q_cnt_v2_WORD word0
#define lpfc_mbx_rq_create_q_id_SHIFT 0
#define lpfc_mbx_rq_create_q_id_MASK 0x0000FFFF
#define lpfc_mbx_rq_create_q_id_WORD word0
#define lpfc_reg_fcfi_vlan_tag_WORD word8
};
+struct lpfc_mbx_reg_fcfi_mrq {
+ uint32_t word1;
+#define lpfc_reg_fcfi_mrq_info_index_SHIFT 0
+#define lpfc_reg_fcfi_mrq_info_index_MASK 0x0000FFFF
+#define lpfc_reg_fcfi_mrq_info_index_WORD word1
+#define lpfc_reg_fcfi_mrq_fcfi_SHIFT 16
+#define lpfc_reg_fcfi_mrq_fcfi_MASK 0x0000FFFF
+#define lpfc_reg_fcfi_mrq_fcfi_WORD word1
+ uint32_t word2;
+#define lpfc_reg_fcfi_mrq_rq_id1_SHIFT 0
+#define lpfc_reg_fcfi_mrq_rq_id1_MASK 0x0000FFFF
+#define lpfc_reg_fcfi_mrq_rq_id1_WORD word2
+#define lpfc_reg_fcfi_mrq_rq_id0_SHIFT 16
+#define lpfc_reg_fcfi_mrq_rq_id0_MASK 0x0000FFFF
+#define lpfc_reg_fcfi_mrq_rq_id0_WORD word2
+ uint32_t word3;
+#define lpfc_reg_fcfi_mrq_rq_id3_SHIFT 0
+#define lpfc_reg_fcfi_mrq_rq_id3_MASK 0x0000FFFF
+#define lpfc_reg_fcfi_mrq_rq_id3_WORD word3
+#define lpfc_reg_fcfi_mrq_rq_id2_SHIFT 16
+#define lpfc_reg_fcfi_mrq_rq_id2_MASK 0x0000FFFF
+#define lpfc_reg_fcfi_mrq_rq_id2_WORD word3
+ uint32_t word4;
+#define lpfc_reg_fcfi_mrq_type_match0_SHIFT 24
+#define lpfc_reg_fcfi_mrq_type_match0_MASK 0x000000FF
+#define lpfc_reg_fcfi_mrq_type_match0_WORD word4
+#define lpfc_reg_fcfi_mrq_type_mask0_SHIFT 16
+#define lpfc_reg_fcfi_mrq_type_mask0_MASK 0x000000FF
+#define lpfc_reg_fcfi_mrq_type_mask0_WORD word4
+#define lpfc_reg_fcfi_mrq_rctl_match0_SHIFT 8
+#define lpfc_reg_fcfi_mrq_rctl_match0_MASK 0x000000FF
+#define lpfc_reg_fcfi_mrq_rctl_match0_WORD word4
+#define lpfc_reg_fcfi_mrq_rctl_mask0_SHIFT 0
+#define lpfc_reg_fcfi_mrq_rctl_mask0_MASK 0x000000FF
+#define lpfc_reg_fcfi_mrq_rctl_mask0_WORD word4
+ uint32_t word5;
+#define lpfc_reg_fcfi_mrq_type_match1_SHIFT 24
+#define lpfc_reg_fcfi_mrq_type_match1_MASK 0x000000FF
+#define lpfc_reg_fcfi_mrq_type_match1_WORD word5
+#define lpfc_reg_fcfi_mrq_type_mask1_SHIFT 16
+#define lpfc_reg_fcfi_mrq_type_mask1_MASK 0x000000FF
+#define lpfc_reg_fcfi_mrq_type_mask1_WORD word5
+#define lpfc_reg_fcfi_mrq_rctl_match1_SHIFT 8
+#define lpfc_reg_fcfi_mrq_rctl_match1_MASK 0x000000FF
+#define lpfc_reg_fcfi_mrq_rctl_match1_WORD word5
+#define lpfc_reg_fcfi_mrq_rctl_mask1_SHIFT 0
+#define lpfc_reg_fcfi_mrq_rctl_mask1_MASK 0x000000FF
+#define lpfc_reg_fcfi_mrq_rctl_mask1_WORD word5
+ uint32_t word6;
+#define lpfc_reg_fcfi_mrq_type_match2_SHIFT 24
+#define lpfc_reg_fcfi_mrq_type_match2_MASK 0x000000FF
+#define lpfc_reg_fcfi_mrq_type_match2_WORD word6
+#define lpfc_reg_fcfi_mrq_type_mask2_SHIFT 16
+#define lpfc_reg_fcfi_mrq_type_mask2_MASK 0x000000FF
+#define lpfc_reg_fcfi_mrq_type_mask2_WORD word6
+#define lpfc_reg_fcfi_mrq_rctl_match2_SHIFT 8
+#define lpfc_reg_fcfi_mrq_rctl_match2_MASK 0x000000FF
+#define lpfc_reg_fcfi_mrq_rctl_match2_WORD word6
+#define lpfc_reg_fcfi_mrq_rctl_mask2_SHIFT 0
+#define lpfc_reg_fcfi_mrq_rctl_mask2_MASK 0x000000FF
+#define lpfc_reg_fcfi_mrq_rctl_mask2_WORD word6
+ uint32_t word7;
+#define lpfc_reg_fcfi_mrq_type_match3_SHIFT 24
+#define lpfc_reg_fcfi_mrq_type_match3_MASK 0x000000FF
+#define lpfc_reg_fcfi_mrq_type_match3_WORD word7
+#define lpfc_reg_fcfi_mrq_type_mask3_SHIFT 16
+#define lpfc_reg_fcfi_mrq_type_mask3_MASK 0x000000FF
+#define lpfc_reg_fcfi_mrq_type_mask3_WORD word7
+#define lpfc_reg_fcfi_mrq_rctl_match3_SHIFT 8
+#define lpfc_reg_fcfi_mrq_rctl_match3_MASK 0x000000FF
+#define lpfc_reg_fcfi_mrq_rctl_match3_WORD word7
+#define lpfc_reg_fcfi_mrq_rctl_mask3_SHIFT 0
+#define lpfc_reg_fcfi_mrq_rctl_mask3_MASK 0x000000FF
+#define lpfc_reg_fcfi_mrq_rctl_mask3_WORD word7
+ uint32_t word8;
+#define lpfc_reg_fcfi_mrq_ptc7_SHIFT 31
+#define lpfc_reg_fcfi_mrq_ptc7_MASK 0x00000001
+#define lpfc_reg_fcfi_mrq_ptc7_WORD word8
+#define lpfc_reg_fcfi_mrq_ptc6_SHIFT 30
+#define lpfc_reg_fcfi_mrq_ptc6_MASK 0x00000001
+#define lpfc_reg_fcfi_mrq_ptc6_WORD word8
+#define lpfc_reg_fcfi_mrq_ptc5_SHIFT 29
+#define lpfc_reg_fcfi_mrq_ptc5_MASK 0x00000001
+#define lpfc_reg_fcfi_mrq_ptc5_WORD word8
+#define lpfc_reg_fcfi_mrq_ptc4_SHIFT 28
+#define lpfc_reg_fcfi_mrq_ptc4_MASK 0x00000001
+#define lpfc_reg_fcfi_mrq_ptc4_WORD word8
+#define lpfc_reg_fcfi_mrq_ptc3_SHIFT 27
+#define lpfc_reg_fcfi_mrq_ptc3_MASK 0x00000001
+#define lpfc_reg_fcfi_mrq_ptc3_WORD word8
+#define lpfc_reg_fcfi_mrq_ptc2_SHIFT 26
+#define lpfc_reg_fcfi_mrq_ptc2_MASK 0x00000001
+#define lpfc_reg_fcfi_mrq_ptc2_WORD word8
+#define lpfc_reg_fcfi_mrq_ptc1_SHIFT 25
+#define lpfc_reg_fcfi_mrq_ptc1_MASK 0x00000001
+#define lpfc_reg_fcfi_mrq_ptc1_WORD word8
+#define lpfc_reg_fcfi_mrq_ptc0_SHIFT 24
+#define lpfc_reg_fcfi_mrq_ptc0_MASK 0x00000001
+#define lpfc_reg_fcfi_mrq_ptc0_WORD word8
+#define lpfc_reg_fcfi_mrq_pt7_SHIFT 23
+#define lpfc_reg_fcfi_mrq_pt7_MASK 0x00000001
+#define lpfc_reg_fcfi_mrq_pt7_WORD word8
+#define lpfc_reg_fcfi_mrq_pt6_SHIFT 22
+#define lpfc_reg_fcfi_mrq_pt6_MASK 0x00000001
+#define lpfc_reg_fcfi_mrq_pt6_WORD word8
+#define lpfc_reg_fcfi_mrq_pt5_SHIFT 21
+#define lpfc_reg_fcfi_mrq_pt5_MASK 0x00000001
+#define lpfc_reg_fcfi_mrq_pt5_WORD word8
+#define lpfc_reg_fcfi_mrq_pt4_SHIFT 20
+#define lpfc_reg_fcfi_mrq_pt4_MASK 0x00000001
+#define lpfc_reg_fcfi_mrq_pt4_WORD word8
+#define lpfc_reg_fcfi_mrq_pt3_SHIFT 19
+#define lpfc_reg_fcfi_mrq_pt3_MASK 0x00000001
+#define lpfc_reg_fcfi_mrq_pt3_WORD word8
+#define lpfc_reg_fcfi_mrq_pt2_SHIFT 18
+#define lpfc_reg_fcfi_mrq_pt2_MASK 0x00000001
+#define lpfc_reg_fcfi_mrq_pt2_WORD word8
+#define lpfc_reg_fcfi_mrq_pt1_SHIFT 17
+#define lpfc_reg_fcfi_mrq_pt1_MASK 0x00000001
+#define lpfc_reg_fcfi_mrq_pt1_WORD word8
+#define lpfc_reg_fcfi_mrq_pt0_SHIFT 16
+#define lpfc_reg_fcfi_mrq_pt0_MASK 0x00000001
+#define lpfc_reg_fcfi_mrq_pt0_WORD word8
+#define lpfc_reg_fcfi_mrq_xmv_SHIFT 15
+#define lpfc_reg_fcfi_mrq_xmv_MASK 0x00000001
+#define lpfc_reg_fcfi_mrq_xmv_WORD word8
+#define lpfc_reg_fcfi_mrq_mode_SHIFT 13
+#define lpfc_reg_fcfi_mrq_mode_MASK 0x00000001
+#define lpfc_reg_fcfi_mrq_mode_WORD word8
+#define lpfc_reg_fcfi_mrq_vv_SHIFT 12
+#define lpfc_reg_fcfi_mrq_vv_MASK 0x00000001
+#define lpfc_reg_fcfi_mrq_vv_WORD word8
+#define lpfc_reg_fcfi_mrq_vlan_tag_SHIFT 0
+#define lpfc_reg_fcfi_mrq_vlan_tag_MASK 0x00000FFF
+#define lpfc_reg_fcfi_mrq_vlan_tag_WORD word8
+ uint32_t word9;
+#define lpfc_reg_fcfi_mrq_policy_SHIFT 12
+#define lpfc_reg_fcfi_mrq_policy_MASK 0x0000000F
+#define lpfc_reg_fcfi_mrq_policy_WORD word9
+#define lpfc_reg_fcfi_mrq_filter_SHIFT 8
+#define lpfc_reg_fcfi_mrq_filter_MASK 0x0000000F
+#define lpfc_reg_fcfi_mrq_filter_WORD word9
+#define lpfc_reg_fcfi_mrq_npairs_SHIFT 0
+#define lpfc_reg_fcfi_mrq_npairs_MASK 0x000000FF
+#define lpfc_reg_fcfi_mrq_npairs_WORD word9
+ uint32_t word10;
+ uint32_t word11;
+ uint32_t word12;
+ uint32_t word13;
+ uint32_t word14;
+ uint32_t word15;
+ uint32_t word16;
+};
+
struct lpfc_mbx_unreg_fcfi {
uint32_t word1_rsv;
uint32_t word2;
#define lpfc_mbx_rq_ftr_rq_perfh_SHIFT 11
#define lpfc_mbx_rq_ftr_rq_perfh_MASK 0x00000001
#define lpfc_mbx_rq_ftr_rq_perfh_WORD word2
+#define lpfc_mbx_rq_ftr_rq_mrqp_SHIFT 16
+#define lpfc_mbx_rq_ftr_rq_mrqp_MASK 0x00000001
+#define lpfc_mbx_rq_ftr_rq_mrqp_WORD word2
uint32_t word3;
#define lpfc_mbx_rq_ftr_rsp_iaab_SHIFT 0
#define lpfc_mbx_rq_ftr_rsp_iaab_MASK 0x00000001
#define lpfc_mbx_rq_ftr_rsp_perfh_SHIFT 11
#define lpfc_mbx_rq_ftr_rsp_perfh_MASK 0x00000001
#define lpfc_mbx_rq_ftr_rsp_perfh_WORD word3
+#define lpfc_mbx_rq_ftr_rsp_mrqp_SHIFT 16
+#define lpfc_mbx_rq_ftr_rsp_mrqp_MASK 0x00000001
+#define lpfc_mbx_rq_ftr_rsp_mrqp_WORD word3
};
struct lpfc_mbx_supp_pages {
#define cfg_mqv_WORD word6
uint32_t word7;
uint32_t word8;
+#define cfg_wqpcnt_SHIFT 0
+#define cfg_wqpcnt_MASK 0x0000000f
+#define cfg_wqpcnt_WORD word8
#define cfg_wqsize_SHIFT 8
#define cfg_wqsize_MASK 0x0000000f
#define cfg_wqsize_WORD word8
#define cfg_wqv_SHIFT 14
#define cfg_wqv_MASK 0x00000003
#define cfg_wqv_WORD word8
+#define cfg_wqpsize_SHIFT 16
+#define cfg_wqpsize_MASK 0x000000ff
+#define cfg_wqpsize_WORD word8
uint32_t word9;
uint32_t word10;
#define cfg_rqv_SHIFT 14
#define cfg_mds_diags_SHIFT 1
#define cfg_mds_diags_MASK 0x00000001
#define cfg_mds_diags_WORD word19
+#define cfg_nvme_SHIFT 3
+#define cfg_nvme_MASK 0x00000001
+#define cfg_nvme_WORD word19
+#define cfg_xib_SHIFT 4
+#define cfg_xib_MASK 0x00000001
+#define cfg_xib_WORD word19
};
#define LPFC_SET_UE_RECOVERY 0x10
struct lpfc_mbx_del_fcf_tbl_entry del_fcf_entry;
struct lpfc_mbx_redisc_fcf_tbl redisc_fcf_tbl;
struct lpfc_mbx_reg_fcfi reg_fcfi;
+ struct lpfc_mbx_reg_fcfi_mrq reg_fcfi_mrq;
struct lpfc_mbx_unreg_fcfi unreg_fcfi;
struct lpfc_mbx_mq_create mq_create;
struct lpfc_mbx_mq_create_ext mq_create_ext;
struct lpfc_mbx_eq_create eq_create;
struct lpfc_mbx_modify_eq_delay eq_delay;
struct lpfc_mbx_cq_create cq_create;
+ struct lpfc_mbx_cq_create_set cq_create_set;
struct lpfc_mbx_wq_create wq_create;
struct lpfc_mbx_rq_create rq_create;
+ struct lpfc_mbx_rq_create_v2 rq_create_v2;
struct lpfc_mbx_mq_destroy mq_destroy;
struct lpfc_mbx_eq_destroy eq_destroy;
struct lpfc_mbx_cq_destroy cq_destroy;
#define wqe_ebde_cnt_SHIFT 0
#define wqe_ebde_cnt_MASK 0x0000000f
#define wqe_ebde_cnt_WORD word10
+#define wqe_nvme_SHIFT 4
+#define wqe_nvme_MASK 0x00000001
+#define wqe_nvme_WORD word10
#define wqe_oas_SHIFT 6
#define wqe_oas_MASK 0x00000001
#define wqe_oas_WORD word10
#define LPFC_ELS_ID_FDISC 2
#define LPFC_ELS_ID_LOGO 1
#define LPFC_ELS_ID_DEFAULT 0
+#define wqe_irsp_SHIFT 4
+#define wqe_irsp_MASK 0x00000001
+#define wqe_irsp_WORD word11
+#define wqe_sup_SHIFT 6
+#define wqe_sup_MASK 0x00000001
+#define wqe_sup_WORD word11
#define wqe_wqec_SHIFT 7
#define wqe_wqec_MASK 0x00000001
#define wqe_wqec_WORD word11
+#define wqe_irsplen_SHIFT 8
+#define wqe_irsplen_MASK 0x0000000f
+#define wqe_irsplen_WORD word11
#define wqe_cqid_SHIFT 16
#define wqe_cqid_MASK 0x0000ffff
#define wqe_cqid_WORD word11
uint32_t max_response_payload_len;
};
+/* Define NVME PRLI request to fabric. NVME is a
+ * fabric-only protocol.
+ * Updated to red-lined v1.08 on Sept 16, 2016
+ */
+struct lpfc_nvme_prli {
+ uint32_t word1;
+ /* The Response Code is defined in the FCP PRLI lpfc_hw.h */
+#define prli_acc_rsp_code_SHIFT 8
+#define prli_acc_rsp_code_MASK 0x0000000f
+#define prli_acc_rsp_code_WORD word1
+#define prli_estabImagePair_SHIFT 13
+#define prli_estabImagePair_MASK 0x00000001
+#define prli_estabImagePair_WORD word1
+#define prli_type_code_ext_SHIFT 16
+#define prli_type_code_ext_MASK 0x000000ff
+#define prli_type_code_ext_WORD word1
+#define prli_type_code_SHIFT 24
+#define prli_type_code_MASK 0x000000ff
+#define prli_type_code_WORD word1
+ uint32_t word_rsvd2;
+ uint32_t word_rsvd3;
+ uint32_t word4;
+#define prli_fba_SHIFT 0
+#define prli_fba_MASK 0x00000001
+#define prli_fba_WORD word4
+#define prli_disc_SHIFT 3
+#define prli_disc_MASK 0x00000001
+#define prli_disc_WORD word4
+#define prli_tgt_SHIFT 4
+#define prli_tgt_MASK 0x00000001
+#define prli_tgt_WORD word4
+#define prli_init_SHIFT 5
+#define prli_init_MASK 0x00000001
+#define prli_init_WORD word4
+#define prli_recov_SHIFT 8
+#define prli_recov_MASK 0x00000001
+#define prli_recov_WORD word4
+ uint32_t word5;
+#define prli_fb_sz_SHIFT 0
+#define prli_fb_sz_MASK 0x0000ffff
+#define prli_fb_sz_WORD word5
+#define LPFC_NVMET_FB_SZ_MAX 65536 /* Driver target mode only. */
+};
+
struct create_xri_wqe {
uint32_t rsrvd[5]; /* words 0-4 */
struct wqe_did wqe_dest; /* word 5 */
uint32_t rsvd_12_15[4]; /* word 12-15 */
};
+struct fcp_trsp64_wqe {
+ struct ulp_bde64 bde;
+ uint32_t response_len;
+ uint32_t rsvd_4_5[2];
+ struct wqe_common wqe_com; /* words 6-11 */
+ uint32_t rsvd_12_15[4]; /* word 12-15 */
+};
+
+struct fcp_tsend64_wqe {
+ struct ulp_bde64 bde;
+ uint32_t payload_offset_len;
+ uint32_t relative_offset;
+ uint32_t reserved;
+ struct wqe_common wqe_com; /* words 6-11 */
+ uint32_t fcp_data_len; /* word 12 */
+ uint32_t rsvd_13_15[3]; /* word 13-15 */
+};
+
+struct fcp_treceive64_wqe {
+ struct ulp_bde64 bde;
+ uint32_t payload_offset_len;
+ uint32_t relative_offset;
+ uint32_t reserved;
+ struct wqe_common wqe_com; /* words 6-11 */
+ uint32_t fcp_data_len; /* word 12 */
+ uint32_t rsvd_13_15[3]; /* word 13-15 */
+};
+#define TXRDY_PAYLOAD_LEN 12
+
union lpfc_wqe {
uint32_t words[16];
struct xmit_els_rsp64_wqe xmit_els_rsp;
struct els_request64_wqe els_req;
struct gen_req64_wqe gen_req;
+ struct fcp_trsp64_wqe fcp_trsp;
+ struct fcp_tsend64_wqe fcp_tsend;
+ struct fcp_treceive64_wqe fcp_treceive;
+
};
union lpfc_wqe128 {
struct fcp_icmnd64_wqe fcp_icmd;
struct fcp_iread64_wqe fcp_iread;
struct fcp_iwrite64_wqe fcp_iwrite;
+ struct fcp_trsp64_wqe fcp_trsp;
+ struct fcp_tsend64_wqe fcp_tsend;
+ struct fcp_treceive64_wqe fcp_treceive;
struct xmit_seq64_wqe xmit_sequence;
struct gen_req64_wqe gen_req;
};
uint8_t revision[32];
};
-#define FCP_COMMAND 0x0
-#define FCP_COMMAND_DATA_OUT 0x1
-#define ELS_COMMAND_NON_FIP 0xC
-#define ELS_COMMAND_FIP 0xD
-#define OTHER_COMMAND 0x8
+/* Defines for WQE command type */
+#define FCP_COMMAND 0x0
+#define NVME_READ_CMD 0x0
+#define FCP_COMMAND_DATA_OUT 0x1
+#define NVME_WRITE_CMD 0x1
+#define FCP_COMMAND_TRECEIVE 0x2
+#define FCP_COMMAND_TRSP 0x3
+#define FCP_COMMAND_TSEND 0x7
+#define OTHER_COMMAND 0x8
+#define ELS_COMMAND_NON_FIP 0xC
+#define ELS_COMMAND_FIP 0xD
+
+#define LPFC_NVME_EMBED_CMD 0x0
+#define LPFC_NVME_EMBED_WRITE 0x1
+#define LPFC_NVME_EMBED_READ 0x2
+
+/* WQE Commands */
+#define CMD_ABORT_XRI_WQE 0x0F
+#define CMD_XMIT_SEQUENCE64_WQE 0x82
+#define CMD_XMIT_BCAST64_WQE 0x84
+#define CMD_ELS_REQUEST64_WQE 0x8A
+#define CMD_XMIT_ELS_RSP64_WQE 0x95
+#define CMD_XMIT_BLS_RSP64_WQE 0x97
+#define CMD_FCP_IWRITE64_WQE 0x98
+#define CMD_FCP_IREAD64_WQE 0x9A
+#define CMD_FCP_ICMND64_WQE 0x9C
+#define CMD_FCP_TSEND64_WQE 0x9F
+#define CMD_FCP_TRECEIVE64_WQE 0xA1
+#define CMD_FCP_TRSP64_WQE 0xA3
+#define CMD_GEN_REQUEST64_WQE 0xC2
+
+#define CMD_WQE_MASK 0xff
+
#define LPFC_FW_DUMP 1
#define LPFC_FW_RESET 2
/*******************************************************************
* This file is part of the Emulex Linux Device Driver for *
* Fibre Channel Host Bus Adapters. *
+ * Copyright (C) 2017 Broadcom. All Rights Reserved. The term *
+ * “Broadcom” refers to Broadcom Limited and/or its subsidiaries. *
* Copyright (C) 2004-2016 Emulex. All rights reserved. *
* EMULEX and SLI are trademarks of Emulex. *
- * www.emulex.com *
+ * www.broadcom.com *
* Portions Copyright (C) 2004-2005 Christoph Hellwig *
* *
* This program is free software; you can redistribute it and/or *
/*******************************************************************
* This file is part of the Emulex Linux Device Driver for *
* Fibre Channel Host Bus Adapters. *
+ * Copyright (C) 2017 Broadcom. All Rights Reserved. The term *
+ * “Broadcom” refers to Broadcom Limited and/or its subsidiaries. *
* Copyright (C) 2004-2016 Emulex. All rights reserved. *
* EMULEX and SLI are trademarks of Emulex. *
- * www.emulex.com *
+ * www.broadcom.com *
* Portions Copyright (C) 2004-2005 Christoph Hellwig *
* *
* This program is free software; you can redistribute it and/or *
#include <linux/firmware.h>
#include <linux/miscdevice.h>
#include <linux/percpu.h>
+#include <linux/msi.h>
#include <scsi/scsi.h>
#include <scsi/scsi_device.h>
#include "lpfc_sli4.h"
#include "lpfc_nl.h"
#include "lpfc_disc.h"
-#include "lpfc_scsi.h"
#include "lpfc.h"
+#include "lpfc_scsi.h"
+#include "lpfc_nvme.h"
#include "lpfc_logmsg.h"
#include "lpfc_crtn.h"
#include "lpfc_vport.h"
static int lpfc_setup_endian_order(struct lpfc_hba *);
static void lpfc_destroy_bootstrap_mbox(struct lpfc_hba *);
static void lpfc_free_els_sgl_list(struct lpfc_hba *);
+static void lpfc_free_nvmet_sgl_list(struct lpfc_hba *);
static void lpfc_init_sgl_list(struct lpfc_hba *);
static int lpfc_init_active_sgl_array(struct lpfc_hba *);
static void lpfc_free_active_sgl(struct lpfc_hba *);
static struct scsi_transport_template *lpfc_transport_template = NULL;
static struct scsi_transport_template *lpfc_vport_transport_template = NULL;
static DEFINE_IDR(lpfc_hba_index);
+#define LPFC_NVMET_BUF_POST 254
/**
* lpfc_config_port_prep - Perform lpfc initialization prior to config port
phba->link_state = LPFC_LINK_DOWN;
/* Only process IOCBs on ELS ring till hba_state is READY */
- if (psli->ring[psli->extra_ring].sli.sli3.cmdringaddr)
- psli->ring[psli->extra_ring].flag |= LPFC_STOP_IOCB_EVENT;
- if (psli->ring[psli->fcp_ring].sli.sli3.cmdringaddr)
- psli->ring[psli->fcp_ring].flag |= LPFC_STOP_IOCB_EVENT;
- if (psli->ring[psli->next_ring].sli.sli3.cmdringaddr)
- psli->ring[psli->next_ring].flag |= LPFC_STOP_IOCB_EVENT;
+ if (psli->sli3_ring[LPFC_EXTRA_RING].sli.sli3.cmdringaddr)
+ psli->sli3_ring[LPFC_EXTRA_RING].flag |= LPFC_STOP_IOCB_EVENT;
+ if (psli->sli3_ring[LPFC_FCP_RING].sli.sli3.cmdringaddr)
+ psli->sli3_ring[LPFC_FCP_RING].flag |= LPFC_STOP_IOCB_EVENT;
/* Post receive buffers for desired rings */
if (phba->sli_rev != 3)
lpfc_sli_hbqbuf_free_all(phba);
else {
/* Cleanup preposted buffers on the ELS ring */
- pring = &psli->ring[LPFC_ELS_RING];
+ pring = &psli->sli3_ring[LPFC_ELS_RING];
spin_lock_irq(&phba->hbalock);
list_splice_init(&pring->postbufq, &buflist);
spin_unlock_irq(&phba->hbalock);
lpfc_hba_clean_txcmplq(struct lpfc_hba *phba)
{
struct lpfc_sli *psli = &phba->sli;
+ struct lpfc_queue *qp = NULL;
struct lpfc_sli_ring *pring;
LIST_HEAD(completions);
int i;
- for (i = 0; i < psli->num_rings; i++) {
- pring = &psli->ring[i];
- if (phba->sli_rev >= LPFC_SLI_REV4)
- spin_lock_irq(&pring->ring_lock);
- else
+ if (phba->sli_rev != LPFC_SLI_REV4) {
+ for (i = 0; i < psli->num_rings; i++) {
+ pring = &psli->sli3_ring[i];
spin_lock_irq(&phba->hbalock);
- /* At this point in time the HBA is either reset or DOA. Either
- * way, nothing should be on txcmplq as it will NEVER complete.
- */
- list_splice_init(&pring->txcmplq, &completions);
- pring->txcmplq_cnt = 0;
-
- if (phba->sli_rev >= LPFC_SLI_REV4)
- spin_unlock_irq(&pring->ring_lock);
- else
+ /* At this point in time the HBA is either reset or DOA
+ * Nothing should be on txcmplq as it will
+ * NEVER complete.
+ */
+ list_splice_init(&pring->txcmplq, &completions);
+ pring->txcmplq_cnt = 0;
spin_unlock_irq(&phba->hbalock);
+ lpfc_sli_abort_iocb_ring(phba, pring);
+ }
/* Cancel all the IOCBs from the completions list */
- lpfc_sli_cancel_iocbs(phba, &completions, IOSTAT_LOCAL_REJECT,
- IOERR_SLI_ABORTED);
+ lpfc_sli_cancel_iocbs(phba, &completions,
+ IOSTAT_LOCAL_REJECT, IOERR_SLI_ABORTED);
+ return;
+ }
+ list_for_each_entry(qp, &phba->sli4_hba.lpfc_wq_list, wq_list) {
+ pring = qp->pring;
+ if (!pring)
+ continue;
+ spin_lock_irq(&pring->ring_lock);
+ list_splice_init(&pring->txcmplq, &completions);
+ pring->txcmplq_cnt = 0;
+ spin_unlock_irq(&pring->ring_lock);
lpfc_sli_abort_iocb_ring(phba, pring);
}
+ /* Cancel all the IOCBs from the completions list */
+ lpfc_sli_cancel_iocbs(phba, &completions,
+ IOSTAT_LOCAL_REJECT, IOERR_SLI_ABORTED);
}
/**
{
struct lpfc_scsi_buf *psb, *psb_next;
LIST_HEAD(aborts);
+ LIST_HEAD(nvme_aborts);
unsigned long iflag = 0;
struct lpfc_sglq *sglq_entry = NULL;
- struct lpfc_sli *psli = &phba->sli;
- struct lpfc_sli_ring *pring;
- lpfc_hba_free_post_buf(phba);
+
+ lpfc_sli_hbqbuf_free_all(phba);
lpfc_hba_clean_txcmplq(phba);
- pring = &psli->ring[LPFC_ELS_RING];
/* At this point in time the HBA is either reset or DOA. Either
* way, nothing should be on lpfc_abts_els_sgl_list, it needs to be
- * on the lpfc_sgl_list so that it can either be freed if the
+ * on the lpfc_els_sgl_list so that it can either be freed if the
* driver is unloading or reposted if the driver is restarting
* the port.
*/
- spin_lock_irq(&phba->hbalock); /* required for lpfc_sgl_list and */
+ spin_lock_irq(&phba->hbalock); /* required for lpfc_els_sgl_list and */
/* scsl_buf_list */
- /* abts_sgl_list_lock required because worker thread uses this
+ /* sgl_list_lock required because worker thread uses this
* list.
*/
- spin_lock(&phba->sli4_hba.abts_sgl_list_lock);
+ spin_lock(&phba->sli4_hba.sgl_list_lock);
list_for_each_entry(sglq_entry,
&phba->sli4_hba.lpfc_abts_els_sgl_list, list)
sglq_entry->state = SGL_FREED;
+ list_for_each_entry(sglq_entry,
+ &phba->sli4_hba.lpfc_abts_nvmet_sgl_list, list)
+ sglq_entry->state = SGL_FREED;
- spin_lock(&pring->ring_lock);
list_splice_init(&phba->sli4_hba.lpfc_abts_els_sgl_list,
- &phba->sli4_hba.lpfc_sgl_list);
- spin_unlock(&pring->ring_lock);
- spin_unlock(&phba->sli4_hba.abts_sgl_list_lock);
+ &phba->sli4_hba.lpfc_els_sgl_list);
+
+ if (phba->sli4_hba.nvme_wq)
+ list_splice_init(&phba->sli4_hba.lpfc_abts_nvmet_sgl_list,
+ &phba->sli4_hba.lpfc_nvmet_sgl_list);
+
+ spin_unlock(&phba->sli4_hba.sgl_list_lock);
/* abts_scsi_buf_list_lock required because worker thread uses this
* list.
*/
- spin_lock(&phba->sli4_hba.abts_scsi_buf_list_lock);
- list_splice_init(&phba->sli4_hba.lpfc_abts_scsi_buf_list,
- &aborts);
- spin_unlock(&phba->sli4_hba.abts_scsi_buf_list_lock);
+ if (phba->cfg_enable_fc4_type & LPFC_ENABLE_FCP) {
+ spin_lock(&phba->sli4_hba.abts_scsi_buf_list_lock);
+ list_splice_init(&phba->sli4_hba.lpfc_abts_scsi_buf_list,
+ &aborts);
+ spin_unlock(&phba->sli4_hba.abts_scsi_buf_list_lock);
+ }
+
+ if (phba->cfg_enable_fc4_type & LPFC_ENABLE_NVME) {
+ spin_lock(&phba->sli4_hba.abts_nvme_buf_list_lock);
+ list_splice_init(&phba->sli4_hba.lpfc_abts_nvme_buf_list,
+ &nvme_aborts);
+ spin_unlock(&phba->sli4_hba.abts_nvme_buf_list_lock);
+ }
+
spin_unlock_irq(&phba->hbalock);
list_for_each_entry_safe(psb, psb_next, &aborts, list) {
list_splice(&aborts, &phba->lpfc_scsi_buf_list_put);
spin_unlock_irqrestore(&phba->scsi_buf_list_put_lock, iflag);
+ list_for_each_entry_safe(psb, psb_next, &nvme_aborts, list) {
+ psb->pCmd = NULL;
+ psb->status = IOSTAT_SUCCESS;
+ }
+ spin_lock_irqsave(&phba->nvme_buf_list_put_lock, iflag);
+ list_splice(&nvme_aborts, &phba->lpfc_nvme_buf_list_put);
+ spin_unlock_irqrestore(&phba->nvme_buf_list_put_lock, iflag);
+
lpfc_sli4_free_sp_events(phba);
return 0;
}
* @phba: pointer to lpfc hba data structure.
*
* This routine is invoked from the worker thread to handle a HBA host
- * attention link event.
+ * attention link event. SLI3 only.
**/
void
lpfc_handle_latt(struct lpfc_hba *phba)
pmb->mbox_cmpl = lpfc_mbx_cmpl_read_topology;
pmb->vport = vport;
/* Block ELS IOCBs until we have processed this mbox command */
- phba->sli.ring[LPFC_ELS_RING].flag |= LPFC_STOP_IOCB_EVENT;
+ phba->sli.sli3_ring[LPFC_ELS_RING].flag |= LPFC_STOP_IOCB_EVENT;
rc = lpfc_sli_issue_mbox (phba, pmb, MBX_NOWAIT);
if (rc == MBX_NOT_FINISHED) {
rc = 4;
return;
lpfc_handle_latt_free_mbuf:
- phba->sli.ring[LPFC_ELS_RING].flag &= ~LPFC_STOP_IOCB_EVENT;
+ phba->sli.sli3_ring[LPFC_ELS_RING].flag &= ~LPFC_STOP_IOCB_EVENT;
lpfc_mbuf_free(phba, mp->virt, mp->phys);
lpfc_handle_latt_free_mp:
kfree(mp);
*
* This routine posts initial receive IOCB buffers to the ELS ring. The
* current number of initial IOCB buffers specified by LPFC_BUF_RING0 is
- * set to 64 IOCBs.
+ * set to 64 IOCBs. SLI3 only.
*
* Return codes
* 0 - success (currently always success)
struct lpfc_sli *psli = &phba->sli;
/* Ring 0, ELS / CT buffers */
- lpfc_post_buffer(phba, &psli->ring[LPFC_ELS_RING], LPFC_BUF_RING0);
+ lpfc_post_buffer(phba, &psli->sli3_ring[LPFC_ELS_RING], LPFC_BUF_RING0);
/* Ring 2 - FCP no buffers needed */
return 0;
lpfc_disc_state_machine(vport, ndlp, NULL,
NLP_EVT_DEVICE_RECOVERY);
+ if (ndlp->nlp_fc4_type & NLP_FC4_NVME) {
+ /* Remove the NVME transport reference now and
+ * continue to remove the node.
+ */
+ lpfc_nlp_put(ndlp);
+ }
+
lpfc_disc_state_machine(vport, ndlp, NULL,
NLP_EVT_DEVICE_RM);
}
lpfc_block_mgmt_io(phba, LPFC_MBX_WAIT);
- if (!lpfc_sli_queue_setup(phba)) {
- lpfc_unblock_mgmt_io(phba);
- return 1;
- }
-
if (phba->sli_rev == LPFC_SLI_REV4) {
if (lpfc_sli4_hba_setup(phba)) { /* Initialize SLI4 HBA */
lpfc_unblock_mgmt_io(phba);
vpis_cleared = true;
spin_unlock_irq(&phba->hbalock);
} else {
+ lpfc_sli_queue_init(phba);
if (lpfc_sli_hba_setup(phba)) { /* Initialize SLI2/SLI3 HBA */
lpfc_unblock_mgmt_io(phba);
return 1;
lpfc_scsi_free(struct lpfc_hba *phba)
{
struct lpfc_scsi_buf *sb, *sb_next;
- struct lpfc_iocbq *io, *io_next;
+
+ if (!(phba->cfg_enable_fc4_type & LPFC_ENABLE_FCP))
+ return;
spin_lock_irq(&phba->hbalock);
list_for_each_entry_safe(sb, sb_next, &phba->lpfc_scsi_buf_list_put,
list) {
list_del(&sb->list);
- pci_pool_free(phba->lpfc_scsi_dma_buf_pool, sb->data,
+ pci_pool_free(phba->lpfc_sg_dma_buf_pool, sb->data,
sb->dma_handle);
kfree(sb);
phba->total_scsi_bufs--;
list_for_each_entry_safe(sb, sb_next, &phba->lpfc_scsi_buf_list_get,
list) {
list_del(&sb->list);
- pci_pool_free(phba->lpfc_scsi_dma_buf_pool, sb->data,
+ pci_pool_free(phba->lpfc_sg_dma_buf_pool, sb->data,
sb->dma_handle);
kfree(sb);
phba->total_scsi_bufs--;
}
spin_unlock(&phba->scsi_buf_list_get_lock);
+ spin_unlock_irq(&phba->hbalock);
+}
+/**
+ * lpfc_nvme_free - Free all the NVME buffers and IOCBs from driver lists
+ * @phba: pointer to lpfc hba data structure.
+ *
+ * This routine is to free all the NVME buffers and IOCBs from the driver
+ * list back to kernel. It is called from lpfc_pci_remove_one to free
+ * the internal resources before the device is removed from the system.
+ **/
+static void
+lpfc_nvme_free(struct lpfc_hba *phba)
+{
+ struct lpfc_nvme_buf *lpfc_ncmd, *lpfc_ncmd_next;
- /* Release all the lpfc_iocbq entries maintained by this host. */
- list_for_each_entry_safe(io, io_next, &phba->lpfc_iocb_list, list) {
- list_del(&io->list);
- kfree(io);
- phba->total_iocbq_bufs--;
- }
+ if (!(phba->cfg_enable_fc4_type & LPFC_ENABLE_NVME))
+ return;
+
+ spin_lock_irq(&phba->hbalock);
+ /* Release all the lpfc_nvme_bufs maintained by this host. */
+ spin_lock(&phba->nvme_buf_list_put_lock);
+ list_for_each_entry_safe(lpfc_ncmd, lpfc_ncmd_next,
+ &phba->lpfc_nvme_buf_list_put, list) {
+ list_del(&lpfc_ncmd->list);
+ pci_pool_free(phba->lpfc_sg_dma_buf_pool, lpfc_ncmd->data,
+ lpfc_ncmd->dma_handle);
+ kfree(lpfc_ncmd);
+ phba->total_nvme_bufs--;
+ }
+ spin_unlock(&phba->nvme_buf_list_put_lock);
+
+ spin_lock(&phba->nvme_buf_list_get_lock);
+ list_for_each_entry_safe(lpfc_ncmd, lpfc_ncmd_next,
+ &phba->lpfc_nvme_buf_list_get, list) {
+ list_del(&lpfc_ncmd->list);
+ pci_pool_free(phba->lpfc_sg_dma_buf_pool, lpfc_ncmd->data,
+ lpfc_ncmd->dma_handle);
+ kfree(lpfc_ncmd);
+ phba->total_nvme_bufs--;
+ }
+ spin_unlock(&phba->nvme_buf_list_get_lock);
spin_unlock_irq(&phba->hbalock);
}
-
/**
- * lpfc_sli4_xri_sgl_update - update xri-sgl sizing and mapping
+ * lpfc_sli4_els_sgl_update - update ELS xri-sgl sizing and mapping
* @phba: pointer to lpfc hba data structure.
*
* This routine first calculates the sizes of the current els and allocated
* 0 - successful (for now, it always returns 0)
**/
int
-lpfc_sli4_xri_sgl_update(struct lpfc_hba *phba)
+lpfc_sli4_els_sgl_update(struct lpfc_hba *phba)
{
struct lpfc_sglq *sglq_entry = NULL, *sglq_entry_next = NULL;
- struct lpfc_scsi_buf *psb = NULL, *psb_next = NULL;
- uint16_t i, lxri, xri_cnt, els_xri_cnt, scsi_xri_cnt;
+ uint16_t i, lxri, xri_cnt, els_xri_cnt;
LIST_HEAD(els_sgl_list);
- LIST_HEAD(scsi_sgl_list);
int rc;
- struct lpfc_sli_ring *pring = &phba->sli.ring[LPFC_ELS_RING];
/*
* update on pci function's els xri-sgl list
*/
els_xri_cnt = lpfc_sli4_get_els_iocb_cnt(phba);
+
if (els_xri_cnt > phba->sli4_hba.els_xri_cnt) {
/* els xri-sgl expanded */
xri_cnt = els_xri_cnt - phba->sli4_hba.els_xri_cnt;
list_add_tail(&sglq_entry->list, &els_sgl_list);
}
spin_lock_irq(&phba->hbalock);
- spin_lock(&pring->ring_lock);
- list_splice_init(&els_sgl_list, &phba->sli4_hba.lpfc_sgl_list);
- spin_unlock(&pring->ring_lock);
+ spin_lock(&phba->sli4_hba.sgl_list_lock);
+ list_splice_init(&els_sgl_list,
+ &phba->sli4_hba.lpfc_els_sgl_list);
+ spin_unlock(&phba->sli4_hba.sgl_list_lock);
spin_unlock_irq(&phba->hbalock);
} else if (els_xri_cnt < phba->sli4_hba.els_xri_cnt) {
/* els xri-sgl shrinked */
"%d to %d\n", phba->sli4_hba.els_xri_cnt,
els_xri_cnt);
spin_lock_irq(&phba->hbalock);
- spin_lock(&pring->ring_lock);
- list_splice_init(&phba->sli4_hba.lpfc_sgl_list, &els_sgl_list);
- spin_unlock(&pring->ring_lock);
- spin_unlock_irq(&phba->hbalock);
+ spin_lock(&phba->sli4_hba.sgl_list_lock);
+ list_splice_init(&phba->sli4_hba.lpfc_els_sgl_list,
+ &els_sgl_list);
/* release extra els sgls from list */
for (i = 0; i < xri_cnt; i++) {
list_remove_head(&els_sgl_list,
sglq_entry, struct lpfc_sglq, list);
if (sglq_entry) {
- lpfc_mbuf_free(phba, sglq_entry->virt,
- sglq_entry->phys);
+ __lpfc_mbuf_free(phba, sglq_entry->virt,
+ sglq_entry->phys);
kfree(sglq_entry);
}
}
- spin_lock_irq(&phba->hbalock);
- spin_lock(&pring->ring_lock);
- list_splice_init(&els_sgl_list, &phba->sli4_hba.lpfc_sgl_list);
- spin_unlock(&pring->ring_lock);
+ list_splice_init(&els_sgl_list,
+ &phba->sli4_hba.lpfc_els_sgl_list);
+ spin_unlock(&phba->sli4_hba.sgl_list_lock);
spin_unlock_irq(&phba->hbalock);
} else
lpfc_printf_log(phba, KERN_INFO, LOG_SLI,
sglq_entry = NULL;
sglq_entry_next = NULL;
list_for_each_entry_safe(sglq_entry, sglq_entry_next,
- &phba->sli4_hba.lpfc_sgl_list, list) {
+ &phba->sli4_hba.lpfc_els_sgl_list, list) {
lxri = lpfc_sli4_next_xritag(phba);
if (lxri == NO_XRI) {
lpfc_printf_log(phba, KERN_ERR, LOG_SLI,
sglq_entry->sli4_lxritag = lxri;
sglq_entry->sli4_xritag = phba->sli4_hba.xri_ids[lxri];
}
+ return 0;
+
+out_free_mem:
+ lpfc_free_els_sgl_list(phba);
+ return rc;
+}
+
+/**
+ * lpfc_sli4_nvmet_sgl_update - update xri-sgl sizing and mapping
+ * @phba: pointer to lpfc hba data structure.
+ *
+ * This routine first calculates the sizes of the current els and allocated
+ * scsi sgl lists, and then goes through all sgls to updates the physical
+ * XRIs assigned due to port function reset. During port initialization, the
+ * current els and allocated scsi sgl lists are 0s.
+ *
+ * Return codes
+ * 0 - successful (for now, it always returns 0)
+ **/
+int
+lpfc_sli4_nvmet_sgl_update(struct lpfc_hba *phba)
+{
+ struct lpfc_sglq *sglq_entry = NULL, *sglq_entry_next = NULL;
+ uint16_t i, lxri, xri_cnt, els_xri_cnt;
+ uint16_t nvmet_xri_cnt, tot_cnt;
+ LIST_HEAD(nvmet_sgl_list);
+ int rc;
/*
- * update on pci function's allocated scsi xri-sgl list
+ * update on pci function's nvmet xri-sgl list
+ */
+ els_xri_cnt = lpfc_sli4_get_els_iocb_cnt(phba);
+ nvmet_xri_cnt = phba->cfg_nvmet_mrq * phba->cfg_nvmet_mrq_post;
+ tot_cnt = phba->sli4_hba.max_cfg_param.max_xri - els_xri_cnt;
+ if (nvmet_xri_cnt > tot_cnt) {
+ phba->cfg_nvmet_mrq_post = tot_cnt / phba->cfg_nvmet_mrq;
+ nvmet_xri_cnt = phba->cfg_nvmet_mrq * phba->cfg_nvmet_mrq_post;
+ lpfc_printf_log(phba, KERN_INFO, LOG_SLI,
+ "6301 NVMET post-sgl count changed to %d\n",
+ phba->cfg_nvmet_mrq_post);
+ }
+
+ if (nvmet_xri_cnt > phba->sli4_hba.nvmet_xri_cnt) {
+ /* els xri-sgl expanded */
+ xri_cnt = nvmet_xri_cnt - phba->sli4_hba.nvmet_xri_cnt;
+ lpfc_printf_log(phba, KERN_INFO, LOG_SLI,
+ "6302 NVMET xri-sgl cnt grew from %d to %d\n",
+ phba->sli4_hba.nvmet_xri_cnt, nvmet_xri_cnt);
+ /* allocate the additional nvmet sgls */
+ for (i = 0; i < xri_cnt; i++) {
+ sglq_entry = kzalloc(sizeof(struct lpfc_sglq),
+ GFP_KERNEL);
+ if (sglq_entry == NULL) {
+ lpfc_printf_log(phba, KERN_ERR, LOG_SLI,
+ "6303 Failure to allocate an "
+ "NVMET sgl entry:%d\n", i);
+ rc = -ENOMEM;
+ goto out_free_mem;
+ }
+ sglq_entry->buff_type = NVMET_BUFF_TYPE;
+ sglq_entry->virt = lpfc_nvmet_buf_alloc(phba, 0,
+ &sglq_entry->phys);
+ if (sglq_entry->virt == NULL) {
+ kfree(sglq_entry);
+ lpfc_printf_log(phba, KERN_ERR, LOG_SLI,
+ "6304 Failure to allocate an "
+ "NVMET buf:%d\n", i);
+ rc = -ENOMEM;
+ goto out_free_mem;
+ }
+ sglq_entry->sgl = sglq_entry->virt;
+ memset(sglq_entry->sgl, 0,
+ phba->cfg_sg_dma_buf_size);
+ sglq_entry->state = SGL_FREED;
+ list_add_tail(&sglq_entry->list, &nvmet_sgl_list);
+ }
+ spin_lock_irq(&phba->hbalock);
+ spin_lock(&phba->sli4_hba.sgl_list_lock);
+ list_splice_init(&nvmet_sgl_list,
+ &phba->sli4_hba.lpfc_nvmet_sgl_list);
+ spin_unlock(&phba->sli4_hba.sgl_list_lock);
+ spin_unlock_irq(&phba->hbalock);
+ } else if (nvmet_xri_cnt < phba->sli4_hba.nvmet_xri_cnt) {
+ /* nvmet xri-sgl shrunk */
+ xri_cnt = phba->sli4_hba.nvmet_xri_cnt - nvmet_xri_cnt;
+ lpfc_printf_log(phba, KERN_INFO, LOG_SLI,
+ "6305 NVMET xri-sgl count decreased from "
+ "%d to %d\n", phba->sli4_hba.nvmet_xri_cnt,
+ nvmet_xri_cnt);
+ spin_lock_irq(&phba->hbalock);
+ spin_lock(&phba->sli4_hba.sgl_list_lock);
+ list_splice_init(&phba->sli4_hba.lpfc_nvmet_sgl_list,
+ &nvmet_sgl_list);
+ /* release extra nvmet sgls from list */
+ for (i = 0; i < xri_cnt; i++) {
+ list_remove_head(&nvmet_sgl_list,
+ sglq_entry, struct lpfc_sglq, list);
+ if (sglq_entry) {
+ lpfc_nvmet_buf_free(phba, sglq_entry->virt,
+ sglq_entry->phys);
+ kfree(sglq_entry);
+ }
+ }
+ list_splice_init(&nvmet_sgl_list,
+ &phba->sli4_hba.lpfc_nvmet_sgl_list);
+ spin_unlock(&phba->sli4_hba.sgl_list_lock);
+ spin_unlock_irq(&phba->hbalock);
+ } else
+ lpfc_printf_log(phba, KERN_INFO, LOG_SLI,
+ "6306 NVMET xri-sgl count unchanged: %d\n",
+ nvmet_xri_cnt);
+ phba->sli4_hba.nvmet_xri_cnt = nvmet_xri_cnt;
+
+ /* update xris to nvmet sgls on the list */
+ sglq_entry = NULL;
+ sglq_entry_next = NULL;
+ list_for_each_entry_safe(sglq_entry, sglq_entry_next,
+ &phba->sli4_hba.lpfc_nvmet_sgl_list, list) {
+ lxri = lpfc_sli4_next_xritag(phba);
+ if (lxri == NO_XRI) {
+ lpfc_printf_log(phba, KERN_ERR, LOG_SLI,
+ "6307 Failed to allocate xri for "
+ "NVMET sgl\n");
+ rc = -ENOMEM;
+ goto out_free_mem;
+ }
+ sglq_entry->sli4_lxritag = lxri;
+ sglq_entry->sli4_xritag = phba->sli4_hba.xri_ids[lxri];
+ }
+ return 0;
+
+out_free_mem:
+ lpfc_free_nvmet_sgl_list(phba);
+ return rc;
+}
+
+/**
+ * lpfc_sli4_scsi_sgl_update - update xri-sgl sizing and mapping
+ * @phba: pointer to lpfc hba data structure.
+ *
+ * This routine first calculates the sizes of the current els and allocated
+ * scsi sgl lists, and then goes through all sgls to updates the physical
+ * XRIs assigned due to port function reset. During port initialization, the
+ * current els and allocated scsi sgl lists are 0s.
+ *
+ * Return codes
+ * 0 - successful (for now, it always returns 0)
+ **/
+int
+lpfc_sli4_scsi_sgl_update(struct lpfc_hba *phba)
+{
+ struct lpfc_scsi_buf *psb, *psb_next;
+ uint16_t i, lxri, els_xri_cnt, scsi_xri_cnt;
+ LIST_HEAD(scsi_sgl_list);
+ int rc;
+
+ /*
+ * update on pci function's els xri-sgl list
*/
+ els_xri_cnt = lpfc_sli4_get_els_iocb_cnt(phba);
phba->total_scsi_bufs = 0;
+ /*
+ * update on pci function's allocated scsi xri-sgl list
+ */
/* maximum number of xris available for scsi buffers */
phba->sli4_hba.scsi_xri_max = phba->sli4_hba.max_cfg_param.max_xri -
els_xri_cnt;
- lpfc_printf_log(phba, KERN_INFO, LOG_SLI,
- "2401 Current allocated SCSI xri-sgl count:%d, "
- "maximum SCSI xri count:%d\n",
- phba->sli4_hba.scsi_xri_cnt,
- phba->sli4_hba.scsi_xri_max);
+ if (!(phba->cfg_enable_fc4_type & LPFC_ENABLE_FCP))
+ return 0;
+
+ if (phba->cfg_enable_fc4_type & LPFC_ENABLE_NVME)
+ phba->sli4_hba.scsi_xri_max = /* Split them up */
+ (phba->sli4_hba.scsi_xri_max *
+ phba->cfg_xri_split) / 100;
spin_lock_irq(&phba->scsi_buf_list_get_lock);
spin_lock(&phba->scsi_buf_list_put_lock);
list_remove_head(&scsi_sgl_list, psb,
struct lpfc_scsi_buf, list);
if (psb) {
- pci_pool_free(phba->lpfc_scsi_dma_buf_pool,
+ pci_pool_free(phba->lpfc_sg_dma_buf_pool,
psb->data, psb->dma_handle);
kfree(psb);
}
INIT_LIST_HEAD(&phba->lpfc_scsi_buf_list_put);
spin_unlock(&phba->scsi_buf_list_put_lock);
spin_unlock_irq(&phba->scsi_buf_list_get_lock);
-
return 0;
out_free_mem:
- lpfc_free_els_sgl_list(phba);
lpfc_scsi_free(phba);
return rc;
}
+/**
+ * lpfc_sli4_nvme_sgl_update - update xri-sgl sizing and mapping
+ * @phba: pointer to lpfc hba data structure.
+ *
+ * This routine first calculates the sizes of the current els and allocated
+ * scsi sgl lists, and then goes through all sgls to updates the physical
+ * XRIs assigned due to port function reset. During port initialization, the
+ * current els and allocated scsi sgl lists are 0s.
+ *
+ * Return codes
+ * 0 - successful (for now, it always returns 0)
+ **/
+int
+lpfc_sli4_nvme_sgl_update(struct lpfc_hba *phba)
+{
+ struct lpfc_nvme_buf *lpfc_ncmd = NULL, *lpfc_ncmd_next = NULL;
+ uint16_t i, lxri, els_xri_cnt;
+ uint16_t nvme_xri_cnt, nvme_xri_max;
+ LIST_HEAD(nvme_sgl_list);
+ int rc;
+
+ phba->total_nvme_bufs = 0;
+
+ if (!(phba->cfg_enable_fc4_type & LPFC_ENABLE_NVME))
+ return 0;
+ /*
+ * update on pci function's allocated nvme xri-sgl list
+ */
+
+ /* maximum number of xris available for nvme buffers */
+ els_xri_cnt = lpfc_sli4_get_els_iocb_cnt(phba);
+ nvme_xri_max = phba->sli4_hba.max_cfg_param.max_xri - els_xri_cnt;
+ phba->sli4_hba.nvme_xri_max = nvme_xri_max;
+ phba->sli4_hba.nvme_xri_max -= phba->sli4_hba.scsi_xri_max;
+
+ lpfc_printf_log(phba, KERN_INFO, LOG_SLI,
+ "6074 Current allocated NVME xri-sgl count:%d, "
+ "maximum NVME xri count:%d\n",
+ phba->sli4_hba.nvme_xri_cnt,
+ phba->sli4_hba.nvme_xri_max);
+
+ spin_lock_irq(&phba->nvme_buf_list_get_lock);
+ spin_lock(&phba->nvme_buf_list_put_lock);
+ list_splice_init(&phba->lpfc_nvme_buf_list_get, &nvme_sgl_list);
+ list_splice(&phba->lpfc_nvme_buf_list_put, &nvme_sgl_list);
+ spin_unlock(&phba->nvme_buf_list_put_lock);
+ spin_unlock_irq(&phba->nvme_buf_list_get_lock);
+
+ if (phba->sli4_hba.nvme_xri_cnt > phba->sli4_hba.nvme_xri_max) {
+ /* max nvme xri shrunk below the allocated nvme buffers */
+ spin_lock_irq(&phba->nvme_buf_list_get_lock);
+ nvme_xri_cnt = phba->sli4_hba.nvme_xri_cnt -
+ phba->sli4_hba.nvme_xri_max;
+ spin_unlock_irq(&phba->nvme_buf_list_get_lock);
+ /* release the extra allocated nvme buffers */
+ for (i = 0; i < nvme_xri_cnt; i++) {
+ list_remove_head(&nvme_sgl_list, lpfc_ncmd,
+ struct lpfc_nvme_buf, list);
+ if (lpfc_ncmd) {
+ pci_pool_free(phba->lpfc_sg_dma_buf_pool,
+ lpfc_ncmd->data,
+ lpfc_ncmd->dma_handle);
+ kfree(lpfc_ncmd);
+ }
+ }
+ spin_lock_irq(&phba->nvme_buf_list_get_lock);
+ phba->sli4_hba.nvme_xri_cnt -= nvme_xri_cnt;
+ spin_unlock_irq(&phba->nvme_buf_list_get_lock);
+ }
+
+ /* update xris associated to remaining allocated nvme buffers */
+ lpfc_ncmd = NULL;
+ lpfc_ncmd_next = NULL;
+ list_for_each_entry_safe(lpfc_ncmd, lpfc_ncmd_next,
+ &nvme_sgl_list, list) {
+ lxri = lpfc_sli4_next_xritag(phba);
+ if (lxri == NO_XRI) {
+ lpfc_printf_log(phba, KERN_ERR, LOG_SLI,
+ "6075 Failed to allocate xri for "
+ "nvme buffer\n");
+ rc = -ENOMEM;
+ goto out_free_mem;
+ }
+ lpfc_ncmd->cur_iocbq.sli4_lxritag = lxri;
+ lpfc_ncmd->cur_iocbq.sli4_xritag = phba->sli4_hba.xri_ids[lxri];
+ }
+ spin_lock_irq(&phba->nvme_buf_list_get_lock);
+ spin_lock(&phba->nvme_buf_list_put_lock);
+ list_splice_init(&nvme_sgl_list, &phba->lpfc_nvme_buf_list_get);
+ INIT_LIST_HEAD(&phba->lpfc_nvme_buf_list_put);
+ spin_unlock(&phba->nvme_buf_list_put_lock);
+ spin_unlock_irq(&phba->nvme_buf_list_get_lock);
+ return 0;
+
+out_free_mem:
+ lpfc_nvme_free(phba);
+ return rc;
+}
+
/**
* lpfc_create_port - Create an FC port
* @phba: pointer to lpfc hba data structure.
lpfc_create_port(struct lpfc_hba *phba, int instance, struct device *dev)
{
struct lpfc_vport *vport;
- struct Scsi_Host *shost;
+ struct Scsi_Host *shost = NULL;
int error = 0;
- if (dev != &phba->pcidev->dev) {
- shost = scsi_host_alloc(&lpfc_vport_template,
- sizeof(struct lpfc_vport));
- } else {
- if (phba->sli_rev == LPFC_SLI_REV4)
- shost = scsi_host_alloc(&lpfc_template,
- sizeof(struct lpfc_vport));
- else
- shost = scsi_host_alloc(&lpfc_template_s3,
+ if (phba->cfg_enable_fc4_type & LPFC_ENABLE_FCP) {
+ if (dev != &phba->pcidev->dev) {
+ shost = scsi_host_alloc(&lpfc_vport_template,
+ sizeof(struct lpfc_vport));
+ } else {
+ if (phba->sli_rev == LPFC_SLI_REV4)
+ shost = scsi_host_alloc(&lpfc_template,
+ sizeof(struct lpfc_vport));
+ else
+ shost = scsi_host_alloc(&lpfc_template_s3,
+ sizeof(struct lpfc_vport));
+ }
+ } else if (phba->cfg_enable_fc4_type & LPFC_ENABLE_NVME) {
+ shost = scsi_host_alloc(&lpfc_template_nvme,
sizeof(struct lpfc_vport));
}
if (!shost)
vport->load_flag |= FC_LOADING;
vport->fc_flag |= FC_VPORT_NEEDS_REG_VPI;
vport->fc_rscn_flush = 0;
-
lpfc_get_vport_cfgparam(vport);
+
shost->unique_id = instance;
shost->max_id = LPFC_MAX_TARGET;
shost->max_lun = vport->cfg_max_luns;
lpfc_els_flush_all_cmd(phba);
/* Block ELS IOCBs until we have done process link event */
- phba->sli.ring[LPFC_ELS_RING].flag |= LPFC_STOP_IOCB_EVENT;
+ phba->sli4_hba.els_wq->pring->flag |= LPFC_STOP_IOCB_EVENT;
/* Update link event statistics */
phba->sli.slistat.link_event++;
lpfc_els_flush_all_cmd(phba);
/* Block ELS IOCBs until we have done process link event */
- phba->sli.ring[LPFC_ELS_RING].flag |= LPFC_STOP_IOCB_EVENT;
+ phba->sli4_hba.els_wq->pring->flag |= LPFC_STOP_IOCB_EVENT;
/* Update link event statistics */
phba->sli.slistat.link_event++;
sprintf(message, "Unqualified optics - Replace with "
"Avago optics for Warranty and Technical "
"Support - Link is%s operational",
- (operational) ? "" : " not");
+ (operational) ? " not" : "");
break;
case LPFC_SLI_EVENT_STATUS_UNCERTIFIED:
sprintf(message, "Uncertified optics - Replace with "
"Avago-certified optics to enable link "
"operation - Link is%s operational",
- (operational) ? "" : " not");
+ (operational) ? " not" : "");
break;
default:
/* firmware is reporting a status we don't know about */
}
/**
- * lpfc_sli_driver_resource_setup - Setup driver internal resources for SLI3 dev.
+ * lpfc_setup_driver_resource_phase1 - Phase1 etup driver internal resources.
* @phba: pointer to lpfc hba data structure.
*
- * This routine is invoked to set up the driver internal resources specific to
- * support the SLI-3 HBA device it attached to.
+ * This routine is invoked to set up the driver internal resources before the
+ * device specific resource setup to support the HBA device it attached to.
*
* Return codes
- * 0 - successful
- * other values - error
+ * 0 - successful
+ * other values - error
**/
static int
-lpfc_sli_driver_resource_setup(struct lpfc_hba *phba)
+lpfc_setup_driver_resource_phase1(struct lpfc_hba *phba)
{
- struct lpfc_sli *psli;
- int rc;
+ struct lpfc_sli *psli = &phba->sli;
/*
- * Initialize timers used by driver
+ * Driver resources common to all SLI revisions
*/
+ atomic_set(&phba->fast_event_count, 0);
+ spin_lock_init(&phba->hbalock);
- /* Heartbeat timer */
- init_timer(&phba->hb_tmofunc);
- phba->hb_tmofunc.function = lpfc_hb_timeout;
- phba->hb_tmofunc.data = (unsigned long)phba;
+ /* Initialize ndlp management spinlock */
+ spin_lock_init(&phba->ndlp_lock);
- psli = &phba->sli;
- /* MBOX heartbeat timer */
- init_timer(&psli->mbox_tmo);
- psli->mbox_tmo.function = lpfc_mbox_timeout;
- psli->mbox_tmo.data = (unsigned long) phba;
- /* FCP polling mode timer */
- init_timer(&phba->fcp_poll_timer);
- phba->fcp_poll_timer.function = lpfc_poll_timeout;
- phba->fcp_poll_timer.data = (unsigned long) phba;
- /* Fabric block timer */
- init_timer(&phba->fabric_block_timer);
- phba->fabric_block_timer.function = lpfc_fabric_block_timeout;
- phba->fabric_block_timer.data = (unsigned long) phba;
- /* EA polling mode timer */
+ INIT_LIST_HEAD(&phba->port_list);
+ INIT_LIST_HEAD(&phba->work_list);
+ init_waitqueue_head(&phba->wait_4_mlo_m_q);
+
+ /* Initialize the wait queue head for the kernel thread */
+ init_waitqueue_head(&phba->work_waitq);
+
+ lpfc_printf_log(phba, KERN_INFO, LOG_INIT,
+ "1403 Protocols supported %s %s %s\n",
+ ((phba->cfg_enable_fc4_type & LPFC_ENABLE_FCP) ?
+ "SCSI" : " "),
+ ((phba->cfg_enable_fc4_type & LPFC_ENABLE_NVME) ?
+ "NVME" : " "),
+ (phba->nvmet_support ? "NVMET" : " "));
+
+ if (phba->cfg_enable_fc4_type & LPFC_ENABLE_FCP) {
+ /* Initialize the scsi buffer list used by driver for scsi IO */
+ spin_lock_init(&phba->scsi_buf_list_get_lock);
+ INIT_LIST_HEAD(&phba->lpfc_scsi_buf_list_get);
+ spin_lock_init(&phba->scsi_buf_list_put_lock);
+ INIT_LIST_HEAD(&phba->lpfc_scsi_buf_list_put);
+ }
+
+ if ((phba->cfg_enable_fc4_type & LPFC_ENABLE_NVME) &&
+ (phba->nvmet_support == 0)) {
+ /* Initialize the NVME buffer list used by driver for NVME IO */
+ spin_lock_init(&phba->nvme_buf_list_get_lock);
+ INIT_LIST_HEAD(&phba->lpfc_nvme_buf_list_get);
+ spin_lock_init(&phba->nvme_buf_list_put_lock);
+ INIT_LIST_HEAD(&phba->lpfc_nvme_buf_list_put);
+ }
+
+ /* Initialize the fabric iocb list */
+ INIT_LIST_HEAD(&phba->fabric_iocb_list);
+
+ /* Initialize list to save ELS buffers */
+ INIT_LIST_HEAD(&phba->elsbuf);
+
+ /* Initialize FCF connection rec list */
+ INIT_LIST_HEAD(&phba->fcf_conn_rec_list);
+
+ /* Initialize OAS configuration list */
+ spin_lock_init(&phba->devicelock);
+ INIT_LIST_HEAD(&phba->luns);
+
+ /* MBOX heartbeat timer */
+ init_timer(&psli->mbox_tmo);
+ psli->mbox_tmo.function = lpfc_mbox_timeout;
+ psli->mbox_tmo.data = (unsigned long) phba;
+ /* Fabric block timer */
+ init_timer(&phba->fabric_block_timer);
+ phba->fabric_block_timer.function = lpfc_fabric_block_timeout;
+ phba->fabric_block_timer.data = (unsigned long) phba;
+ /* EA polling mode timer */
init_timer(&phba->eratt_poll);
phba->eratt_poll.function = lpfc_poll_eratt;
phba->eratt_poll.data = (unsigned long) phba;
+ /* Heartbeat timer */
+ init_timer(&phba->hb_tmofunc);
+ phba->hb_tmofunc.function = lpfc_hb_timeout;
+ phba->hb_tmofunc.data = (unsigned long)phba;
+
+ return 0;
+}
+
+/**
+ * lpfc_sli_driver_resource_setup - Setup driver internal resources for SLI3 dev
+ * @phba: pointer to lpfc hba data structure.
+ *
+ * This routine is invoked to set up the driver internal resources specific to
+ * support the SLI-3 HBA device it attached to.
+ *
+ * Return codes
+ * 0 - successful
+ * other values - error
+ **/
+static int
+lpfc_sli_driver_resource_setup(struct lpfc_hba *phba)
+{
+ int rc;
+
+ /*
+ * Initialize timers used by driver
+ */
+
+ /* FCP polling mode timer */
+ init_timer(&phba->fcp_poll_timer);
+ phba->fcp_poll_timer.function = lpfc_poll_timeout;
+ phba->fcp_poll_timer.data = (unsigned long) phba;
/* Host attention work mask setup */
phba->work_ha_mask = (HA_ERATT | HA_MBATT | HA_LATT);
/* Get all the module params for configuring this host */
lpfc_get_cfgparam(phba);
+ /* Set up phase-1 common device driver resources */
+
+ rc = lpfc_setup_driver_resource_phase1(phba);
+ if (rc)
+ return -ENODEV;
+
if (phba->pcidev->device == PCI_DEVICE_ID_HORNET) {
phba->menlo_flag |= HBA_MENLO_SUPPORT;
/* check for menlo minimum sg count */
phba->cfg_sg_seg_cnt = LPFC_DEFAULT_MENLO_SG_SEG_CNT;
}
- if (!phba->sli.ring)
- phba->sli.ring = kzalloc(LPFC_SLI3_MAX_RING *
+ if (!phba->sli.sli3_ring)
+ phba->sli.sli3_ring = kzalloc(LPFC_SLI3_MAX_RING *
sizeof(struct lpfc_sli_ring), GFP_KERNEL);
- if (!phba->sli.ring)
+ if (!phba->sli.sli3_ring)
return -ENOMEM;
/*
* Initialize the SLI Layer to run with lpfc HBAs.
*/
lpfc_sli_setup(phba);
- lpfc_sli_queue_setup(phba);
+ lpfc_sli_queue_init(phba);
/* Allocate device driver memory */
if (lpfc_mem_alloc(phba, BPL_ALIGN_SZ))
static int
lpfc_sli4_driver_resource_setup(struct lpfc_hba *phba)
{
- struct lpfc_vector_map_info *cpup;
- struct lpfc_sli *psli;
LPFC_MBOXQ_t *mboxq;
- int rc, i, hbq_count, max_buf_size;
+ MAILBOX_t *mb;
+ int rc, i, max_buf_size;
uint8_t pn_page[LPFC_MAX_SUPPORTED_PAGES] = {0};
struct lpfc_mqe *mqe;
int longs;
int fof_vectors = 0;
+ uint64_t wwn;
+
+ phba->sli4_hba.num_online_cpu = num_online_cpus();
+ phba->sli4_hba.num_present_cpu = lpfc_present_cpu;
+ phba->sli4_hba.curr_disp_cpu = 0;
/* Get all the module params for configuring this host */
lpfc_get_cfgparam(phba);
+ /* Set up phase-1 common device driver resources */
+ rc = lpfc_setup_driver_resource_phase1(phba);
+ if (rc)
+ return -ENODEV;
+
/* Before proceed, wait for POST done and device ready */
rc = lpfc_sli4_post_status_check(phba);
if (rc)
* Initialize timers used by driver
*/
- /* Heartbeat timer */
- init_timer(&phba->hb_tmofunc);
- phba->hb_tmofunc.function = lpfc_hb_timeout;
- phba->hb_tmofunc.data = (unsigned long)phba;
init_timer(&phba->rrq_tmr);
phba->rrq_tmr.function = lpfc_rrq_timeout;
phba->rrq_tmr.data = (unsigned long)phba;
- psli = &phba->sli;
- /* MBOX heartbeat timer */
- init_timer(&psli->mbox_tmo);
- psli->mbox_tmo.function = lpfc_mbox_timeout;
- psli->mbox_tmo.data = (unsigned long) phba;
- /* Fabric block timer */
- init_timer(&phba->fabric_block_timer);
- phba->fabric_block_timer.function = lpfc_fabric_block_timeout;
- phba->fabric_block_timer.data = (unsigned long) phba;
- /* EA polling mode timer */
- init_timer(&phba->eratt_poll);
- phba->eratt_poll.function = lpfc_poll_eratt;
- phba->eratt_poll.data = (unsigned long) phba;
/* FCF rediscover timer */
init_timer(&phba->fcf.redisc_wait);
phba->fcf.redisc_wait.function = lpfc_sli4_fcf_redisc_wait_tmo;
/*
* For SLI4, instead of using ring 0 (LPFC_FCP_RING) for FCP commands
- * we will associate a new ring, for each FCP fastpath EQ/CQ/WQ tuple.
+ * we will associate a new ring, for each EQ/CQ/WQ tuple.
+ * The WQ create will allocate the ring.
*/
- if (!phba->sli.ring)
- phba->sli.ring = kzalloc(
- (LPFC_SLI3_MAX_RING + phba->cfg_fcp_io_channel) *
- sizeof(struct lpfc_sli_ring), GFP_KERNEL);
- if (!phba->sli.ring)
- return -ENOMEM;
/*
* It doesn't matter what family our adapter is in, we are
phba->cfg_sg_seg_cnt = LPFC_MAX_SGL_SEG_CNT - 2;
/*
- * Since lpfc_sg_seg_cnt is module parameter, the sg_dma_buf_size
- * used to create the sg_dma_buf_pool must be dynamically calculated.
+ * Since lpfc_sg_seg_cnt is module param, the sg_dma_buf_size
+ * used to create the sg_dma_buf_pool must be calculated.
*/
-
if (phba->cfg_enable_bg) {
/*
- * The scsi_buf for a T10-DIF I/O will hold the FCP cmnd,
- * the FCP rsp, and a SGE for each. Sice we have no control
- * over how many protection data segments the SCSI Layer
+ * The scsi_buf for a T10-DIF I/O holds the FCP cmnd,
+ * the FCP rsp, and a SGE. Sice we have no control
+ * over how many protection segments the SCSI Layer
* will hand us (ie: there could be one for every block
- * in the IO), we just allocate enough SGEs to accomidate
- * our max amount and we need to limit lpfc_sg_seg_cnt to
- * minimize the risk of running out.
+ * in the IO), just allocate enough SGEs to accomidate
+ * our max amount and we need to limit lpfc_sg_seg_cnt
+ * to minimize the risk of running out.
*/
phba->cfg_sg_dma_buf_size = sizeof(struct fcp_cmnd) +
- sizeof(struct fcp_rsp) + max_buf_size;
+ sizeof(struct fcp_rsp) + max_buf_size;
/* Total SGEs for scsi_sg_list and scsi_sg_prot_list */
phba->cfg_total_seg_cnt = LPFC_MAX_SGL_SEG_CNT;
if (phba->cfg_sg_seg_cnt > LPFC_MAX_SG_SLI4_SEG_CNT_DIF)
- phba->cfg_sg_seg_cnt = LPFC_MAX_SG_SLI4_SEG_CNT_DIF;
+ phba->cfg_sg_seg_cnt =
+ LPFC_MAX_SG_SLI4_SEG_CNT_DIF;
} else {
/*
- * The scsi_buf for a regular I/O will hold the FCP cmnd,
+ * The scsi_buf for a regular I/O holds the FCP cmnd,
* the FCP rsp, a SGE for each, and a SGE for up to
* cfg_sg_seg_cnt data segments.
*/
phba->cfg_sg_dma_buf_size = sizeof(struct fcp_cmnd) +
- sizeof(struct fcp_rsp) +
- ((phba->cfg_sg_seg_cnt + 2) * sizeof(struct sli4_sge));
+ sizeof(struct fcp_rsp) +
+ ((phba->cfg_sg_seg_cnt + 2) *
+ sizeof(struct sli4_sge));
/* Total SGEs for scsi_sg_list */
phba->cfg_total_seg_cnt = phba->cfg_sg_seg_cnt + 2;
+
/*
- * NOTE: if (phba->cfg_sg_seg_cnt + 2) <= 256 we only need
- * to post 1 page for the SGL.
+ * NOTE: if (phba->cfg_sg_seg_cnt + 2) <= 256 we only
+ * need to post 1 page for the SGL.
*/
}
phba->cfg_total_seg_cnt);
/* Initialize buffer queue management fields */
- hbq_count = lpfc_sli_hbq_count();
- for (i = 0; i < hbq_count; ++i)
- INIT_LIST_HEAD(&phba->hbqs[i].hbq_buffer_list);
- INIT_LIST_HEAD(&phba->rb_pend_list);
+ INIT_LIST_HEAD(&phba->hbqs[LPFC_ELS_HBQ].hbq_buffer_list);
phba->hbqs[LPFC_ELS_HBQ].hbq_alloc_buffer = lpfc_sli4_rb_alloc;
phba->hbqs[LPFC_ELS_HBQ].hbq_free_buffer = lpfc_sli4_rb_free;
/*
* Initialize the SLI Layer to run with lpfc SLI4 HBAs.
*/
- /* Initialize the Abort scsi buffer list used by driver */
- spin_lock_init(&phba->sli4_hba.abts_scsi_buf_list_lock);
- INIT_LIST_HEAD(&phba->sli4_hba.lpfc_abts_scsi_buf_list);
+ if (phba->cfg_enable_fc4_type & LPFC_ENABLE_FCP) {
+ /* Initialize the Abort scsi buffer list used by driver */
+ spin_lock_init(&phba->sli4_hba.abts_scsi_buf_list_lock);
+ INIT_LIST_HEAD(&phba->sli4_hba.lpfc_abts_scsi_buf_list);
+ }
+
+ if (phba->cfg_enable_fc4_type & LPFC_ENABLE_NVME) {
+ /* Initialize the Abort nvme buffer list used by driver */
+ spin_lock_init(&phba->sli4_hba.abts_nvme_buf_list_lock);
+ INIT_LIST_HEAD(&phba->sli4_hba.lpfc_abts_nvme_buf_list);
+ }
+
/* This abort list used by worker thread */
- spin_lock_init(&phba->sli4_hba.abts_sgl_list_lock);
+ spin_lock_init(&phba->sli4_hba.sgl_list_lock);
+ spin_lock_init(&phba->sli4_hba.nvmet_io_lock);
/*
* Initialize driver internal slow-path work queues
/* initialize optic_state to 0xFF */
phba->sli4_hba.lnk_info.optic_state = 0xff;
- /* Initialize the driver internal SLI layer lists. */
- lpfc_sli_setup(phba);
- lpfc_sli_queue_setup(phba);
-
/* Allocate device driver memory */
rc = lpfc_mem_alloc(phba, SGL_ALIGN_SZ);
if (rc)
if (bf_get(lpfc_sli_intf_if_type, &phba->sli4_hba.sli_intf) ==
LPFC_SLI_INTF_IF_TYPE_2) {
rc = lpfc_pci_function_reset(phba);
- if (unlikely(rc))
- return -ENODEV;
+ if (unlikely(rc)) {
+ rc = -ENODEV;
+ goto out_free_mem;
+ }
phba->temp_sensor_support = 1;
}
goto out_free_bsmbx;
}
+ /* Check for NVMET being configured */
+ phba->nvmet_support = 0;
+ if (lpfc_enable_nvmet_cnt) {
+
+ /* First get WWN of HBA instance */
+ lpfc_read_nv(phba, mboxq);
+ rc = lpfc_sli_issue_mbox(phba, mboxq, MBX_POLL);
+ if (rc != MBX_SUCCESS) {
+ lpfc_printf_log(phba, KERN_ERR, LOG_SLI,
+ "6016 Mailbox failed , mbxCmd x%x "
+ "READ_NV, mbxStatus x%x\n",
+ bf_get(lpfc_mqe_command, &mboxq->u.mqe),
+ bf_get(lpfc_mqe_status, &mboxq->u.mqe));
+ rc = -EIO;
+ goto out_free_bsmbx;
+ }
+ mb = &mboxq->u.mb;
+ memcpy(&wwn, (char *)mb->un.varRDnvp.nodename,
+ sizeof(uint64_t));
+ wwn = cpu_to_be64(wwn);
+ phba->sli4_hba.wwnn.u.name = wwn;
+ memcpy(&wwn, (char *)mb->un.varRDnvp.portname,
+ sizeof(uint64_t));
+ /* wwn is WWPN of HBA instance */
+ wwn = cpu_to_be64(wwn);
+ phba->sli4_hba.wwpn.u.name = wwn;
+
+ /* Check to see if it matches any module parameter */
+ for (i = 0; i < lpfc_enable_nvmet_cnt; i++) {
+ if (wwn == lpfc_enable_nvmet[i]) {
+ lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+ "6017 NVME Target %016llx\n",
+ wwn);
+ phba->nvmet_support = 1; /* a match */
+ }
+ }
+ }
+
+ lpfc_nvme_mod_param_dep(phba);
+
/* Get the Supported Pages if PORT_CAPABILITIES is supported by port. */
lpfc_supported_pages(mboxq);
rc = lpfc_sli_issue_mbox(phba, mboxq, MBX_POLL);
lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
"2999 Unsupported SLI4 Parameters "
"Extents and RPI headers enabled.\n");
- goto out_free_bsmbx;
}
+ mempool_free(mboxq, phba->mbox_mem_pool);
+ goto out_free_bsmbx;
}
+
mempool_free(mboxq, phba->mbox_mem_pool);
/* Verify OAS is supported */
goto out_remove_rpi_hdrs;
}
- phba->sli4_hba.fcp_eq_hdl =
- kzalloc((sizeof(struct lpfc_fcp_eq_hdl) *
- (fof_vectors + phba->cfg_fcp_io_channel)),
- GFP_KERNEL);
- if (!phba->sli4_hba.fcp_eq_hdl) {
+ phba->sli4_hba.hba_eq_hdl = kcalloc(fof_vectors + phba->io_channel_irqs,
+ sizeof(struct lpfc_hba_eq_hdl),
+ GFP_KERNEL);
+ if (!phba->sli4_hba.hba_eq_hdl) {
lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
"2572 Failed allocate memory for "
"fast-path per-EQ handle array\n");
goto out_free_fcf_rr_bmask;
}
- phba->sli4_hba.msix_entries = kzalloc((sizeof(struct msix_entry) *
- (fof_vectors +
- phba->cfg_fcp_io_channel)), GFP_KERNEL);
- if (!phba->sli4_hba.msix_entries) {
- lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
- "2573 Failed allocate memory for msi-x "
- "interrupt vector entries\n");
- rc = -ENOMEM;
- goto out_free_fcp_eq_hdl;
- }
-
- phba->sli4_hba.cpu_map = kzalloc((sizeof(struct lpfc_vector_map_info) *
- phba->sli4_hba.num_present_cpu),
- GFP_KERNEL);
+ phba->sli4_hba.cpu_map = kcalloc(phba->sli4_hba.num_present_cpu,
+ sizeof(struct lpfc_vector_map_info),
+ GFP_KERNEL);
if (!phba->sli4_hba.cpu_map) {
lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
"3327 Failed allocate memory for msi-x "
"interrupt vector mapping\n");
rc = -ENOMEM;
- goto out_free_msix;
+ goto out_free_hba_eq_hdl;
}
if (lpfc_used_cpu == NULL) {
- lpfc_used_cpu = kzalloc((sizeof(uint16_t) * lpfc_present_cpu),
- GFP_KERNEL);
+ lpfc_used_cpu = kcalloc(lpfc_present_cpu, sizeof(uint16_t),
+ GFP_KERNEL);
if (!lpfc_used_cpu) {
lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
"3335 Failed allocate memory for msi-x "
"interrupt vector mapping\n");
kfree(phba->sli4_hba.cpu_map);
rc = -ENOMEM;
- goto out_free_msix;
+ goto out_free_hba_eq_hdl;
}
for (i = 0; i < lpfc_present_cpu; i++)
lpfc_used_cpu[i] = LPFC_VECTOR_MAP_EMPTY;
}
- /* Initialize io channels for round robin */
- cpup = phba->sli4_hba.cpu_map;
- rc = 0;
- for (i = 0; i < phba->sli4_hba.num_present_cpu; i++) {
- cpup->channel_id = rc;
- rc++;
- if (rc >= phba->cfg_fcp_io_channel)
- rc = 0;
- }
-
/*
* Enable sr-iov virtual functions if supported and configured
* through the module parameter.
return 0;
-out_free_msix:
- kfree(phba->sli4_hba.msix_entries);
-out_free_fcp_eq_hdl:
- kfree(phba->sli4_hba.fcp_eq_hdl);
+out_free_hba_eq_hdl:
+ kfree(phba->sli4_hba.hba_eq_hdl);
out_free_fcf_rr_bmask:
kfree(phba->fcf.fcf_rr_bmask);
out_remove_rpi_hdrs:
phba->sli4_hba.num_online_cpu = 0;
phba->sli4_hba.curr_disp_cpu = 0;
- /* Free memory allocated for msi-x interrupt vector entries */
- kfree(phba->sli4_hba.msix_entries);
-
/* Free memory allocated for fast-path work queue handles */
- kfree(phba->sli4_hba.fcp_eq_hdl);
+ kfree(phba->sli4_hba.hba_eq_hdl);
/* Free the allocated rpi headers. */
lpfc_sli4_remove_rpi_hdrs(phba);
/* Free the ELS sgl list */
lpfc_free_active_sgl(phba);
lpfc_free_els_sgl_list(phba);
+ lpfc_free_nvmet_sgl_list(phba);
/* Free the completion queue EQ event pool */
lpfc_sli4_cq_event_release_all(phba);
return 0;
}
-/**
- * lpfc_setup_driver_resource_phase1 - Phase1 etup driver internal resources.
- * @phba: pointer to lpfc hba data structure.
- *
- * This routine is invoked to set up the driver internal resources before the
- * device specific resource setup to support the HBA device it attached to.
- *
- * Return codes
- * 0 - successful
- * other values - error
- **/
-static int
-lpfc_setup_driver_resource_phase1(struct lpfc_hba *phba)
-{
- /*
- * Driver resources common to all SLI revisions
- */
- atomic_set(&phba->fast_event_count, 0);
- spin_lock_init(&phba->hbalock);
-
- /* Initialize ndlp management spinlock */
- spin_lock_init(&phba->ndlp_lock);
-
- INIT_LIST_HEAD(&phba->port_list);
- INIT_LIST_HEAD(&phba->work_list);
- init_waitqueue_head(&phba->wait_4_mlo_m_q);
-
- /* Initialize the wait queue head for the kernel thread */
- init_waitqueue_head(&phba->work_waitq);
-
- /* Initialize the scsi buffer list used by driver for scsi IO */
- spin_lock_init(&phba->scsi_buf_list_get_lock);
- INIT_LIST_HEAD(&phba->lpfc_scsi_buf_list_get);
- spin_lock_init(&phba->scsi_buf_list_put_lock);
- INIT_LIST_HEAD(&phba->lpfc_scsi_buf_list_put);
-
- /* Initialize the fabric iocb list */
- INIT_LIST_HEAD(&phba->fabric_iocb_list);
-
- /* Initialize list to save ELS buffers */
- INIT_LIST_HEAD(&phba->elsbuf);
-
- /* Initialize FCF connection rec list */
- INIT_LIST_HEAD(&phba->fcf_conn_rec_list);
-
- /* Initialize OAS configuration list */
- spin_lock_init(&phba->devicelock);
- INIT_LIST_HEAD(&phba->luns);
-
- return 0;
-}
-
/**
* lpfc_setup_driver_resource_phase2 - Phase2 setup driver internal resources.
* @phba: pointer to lpfc hba data structure.
lpfc_free_els_sgl_list(struct lpfc_hba *phba)
{
LIST_HEAD(sglq_list);
- struct lpfc_sli_ring *pring = &phba->sli.ring[LPFC_ELS_RING];
/* Retrieve all els sgls from driver list */
spin_lock_irq(&phba->hbalock);
- spin_lock(&pring->ring_lock);
- list_splice_init(&phba->sli4_hba.lpfc_sgl_list, &sglq_list);
- spin_unlock(&pring->ring_lock);
+ spin_lock(&phba->sli4_hba.sgl_list_lock);
+ list_splice_init(&phba->sli4_hba.lpfc_els_sgl_list, &sglq_list);
+ spin_unlock(&phba->sli4_hba.sgl_list_lock);
spin_unlock_irq(&phba->hbalock);
/* Now free the sgl list */
lpfc_free_sgl_list(phba, &sglq_list);
}
+/**
+ * lpfc_free_nvmet_sgl_list - Free nvmet sgl list.
+ * @phba: pointer to lpfc hba data structure.
+ *
+ * This routine is invoked to free the driver's nvmet sgl list and memory.
+ **/
+static void
+lpfc_free_nvmet_sgl_list(struct lpfc_hba *phba)
+{
+ struct lpfc_sglq *sglq_entry = NULL, *sglq_next = NULL;
+ LIST_HEAD(sglq_list);
+
+ /* Retrieve all nvmet sgls from driver list */
+ spin_lock_irq(&phba->hbalock);
+ spin_lock(&phba->sli4_hba.sgl_list_lock);
+ list_splice_init(&phba->sli4_hba.lpfc_nvmet_sgl_list, &sglq_list);
+ spin_unlock(&phba->sli4_hba.sgl_list_lock);
+ spin_unlock_irq(&phba->hbalock);
+
+ /* Now free the sgl list */
+ list_for_each_entry_safe(sglq_entry, sglq_next, &sglq_list, list) {
+ list_del(&sglq_entry->list);
+ lpfc_nvmet_buf_free(phba, sglq_entry->virt, sglq_entry->phys);
+ kfree(sglq_entry);
+ }
+}
+
/**
* lpfc_init_active_sgl_array - Allocate the buf to track active ELS XRIs.
* @phba: pointer to lpfc hba data structure.
lpfc_init_sgl_list(struct lpfc_hba *phba)
{
/* Initialize and populate the sglq list per host/VF. */
- INIT_LIST_HEAD(&phba->sli4_hba.lpfc_sgl_list);
+ INIT_LIST_HEAD(&phba->sli4_hba.lpfc_els_sgl_list);
INIT_LIST_HEAD(&phba->sli4_hba.lpfc_abts_els_sgl_list);
+ INIT_LIST_HEAD(&phba->sli4_hba.lpfc_nvmet_sgl_list);
+ INIT_LIST_HEAD(&phba->sli4_hba.lpfc_abts_nvmet_sgl_list);
/* els xri-sgl book keeping */
phba->sli4_hba.els_xri_cnt = 0;
/* scsi xri-buffer book keeping */
phba->sli4_hba.scsi_xri_cnt = 0;
+
+ /* nvme xri-buffer book keeping */
+ phba->sli4_hba.nvme_xri_cnt = 0;
}
/**
/* Release the driver assigned board number */
idr_remove(&lpfc_hba_index, phba->brd_no);
- /* Free memory allocated with sli rings */
- kfree(phba->sli.ring);
- phba->sli.ring = NULL;
+ /* Free memory allocated with sli3 rings */
+ kfree(phba->sli.sli3_ring);
+ phba->sli.sli3_ring = NULL;
kfree(phba);
return;
shost = lpfc_shost_from_vport(vport);
phba->pport = vport;
+
+ if (phba->nvmet_support) {
+ /* Only 1 vport (pport) will support NVME target */
+ if (phba->txrdy_payload_pool == NULL) {
+ phba->txrdy_payload_pool = pci_pool_create(
+ "txrdy_pool", phba->pcidev,
+ TXRDY_PAYLOAD_LEN, 16, 0);
+ if (phba->txrdy_payload_pool) {
+ phba->targetport = NULL;
+ phba->cfg_enable_fc4_type = LPFC_ENABLE_NVME;
+ lpfc_printf_log(phba, KERN_INFO,
+ LOG_INIT | LOG_NVME_DISC,
+ "6076 NVME Target Found\n");
+ }
+ }
+ }
+
lpfc_debugfs_initialize(vport);
/* Put reference to SCSI host to driver's device private data */
pci_set_drvdata(phba->pcidev, shost);
memset(phba->hbqslimp.virt, 0, lpfc_sli_hbq_size());
- INIT_LIST_HEAD(&phba->rb_pend_list);
-
phba->MBslimaddr = phba->slim_memmap_p;
phba->HAregaddr = phba->ctrl_regs_memmap_p + HA_REG_OFFSET;
phba->CAregaddr = phba->ctrl_regs_memmap_p + CA_REG_OFFSET;
"VPI(B:%d M:%d) "
"VFI(B:%d M:%d) "
"RPI(B:%d M:%d) "
- "FCFI(Count:%d)\n",
+ "FCFI:%d EQ:%d CQ:%d WQ:%d RQ:%d\n",
phba->sli4_hba.extents_in_use,
phba->sli4_hba.max_cfg_param.xri_base,
phba->sli4_hba.max_cfg_param.max_xri,
phba->sli4_hba.max_cfg_param.max_vfi,
phba->sli4_hba.max_cfg_param.rpi_base,
phba->sli4_hba.max_cfg_param.max_rpi,
- phba->sli4_hba.max_cfg_param.max_fcfi);
+ phba->sli4_hba.max_cfg_param.max_fcfi,
+ phba->sli4_hba.max_cfg_param.max_eq,
+ phba->sli4_hba.max_cfg_param.max_cq,
+ phba->sli4_hba.max_cfg_param.max_wq,
+ phba->sli4_hba.max_cfg_param.max_rq);
+
}
if (rc)
}
/**
- * lpfc_sli4_queue_verify - Verify and update EQ and CQ counts
+ * lpfc_sli4_queue_verify - Verify and update EQ counts
* @phba: pointer to lpfc hba data structure.
*
- * This routine is invoked to check the user settable queue counts for EQs and
- * CQs. after this routine is called the counts will be set to valid values that
+ * This routine is invoked to check the user settable queue counts for EQs.
+ * After this routine is called the counts will be set to valid values that
* adhere to the constraints of the system's interrupt vectors and the port's
* queue resources.
*
static int
lpfc_sli4_queue_verify(struct lpfc_hba *phba)
{
- int cfg_fcp_io_channel;
- uint32_t cpu;
- uint32_t i = 0;
+ int io_channel;
int fof_vectors = phba->cfg_fof ? 1 : 0;
/*
*/
/* Sanity check on HBA EQ parameters */
- cfg_fcp_io_channel = phba->cfg_fcp_io_channel;
-
- /* It doesn't make sense to have more io channels then online CPUs */
- for_each_present_cpu(cpu) {
- if (cpu_online(cpu))
- i++;
- }
- phba->sli4_hba.num_online_cpu = i;
- phba->sli4_hba.num_present_cpu = lpfc_present_cpu;
- phba->sli4_hba.curr_disp_cpu = 0;
+ io_channel = phba->io_channel_irqs;
- if (i < cfg_fcp_io_channel) {
+ if (phba->sli4_hba.num_online_cpu < io_channel) {
lpfc_printf_log(phba,
KERN_ERR, LOG_INIT,
"3188 Reducing IO channels to match number of "
"online CPUs: from %d to %d\n",
- cfg_fcp_io_channel, i);
- cfg_fcp_io_channel = i;
+ io_channel, phba->sli4_hba.num_online_cpu);
+ io_channel = phba->sli4_hba.num_online_cpu;
}
- if (cfg_fcp_io_channel + fof_vectors >
- phba->sli4_hba.max_cfg_param.max_eq) {
- if (phba->sli4_hba.max_cfg_param.max_eq <
- LPFC_FCP_IO_CHAN_MIN) {
- lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
- "2574 Not enough EQs (%d) from the "
- "pci function for supporting FCP "
- "EQs (%d)\n",
- phba->sli4_hba.max_cfg_param.max_eq,
- phba->cfg_fcp_io_channel);
- goto out_error;
- }
+ if (io_channel + fof_vectors > phba->sli4_hba.max_cfg_param.max_eq) {
lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
"2575 Reducing IO channels to match number of "
"available EQs: from %d to %d\n",
- cfg_fcp_io_channel,
+ io_channel,
phba->sli4_hba.max_cfg_param.max_eq);
- cfg_fcp_io_channel = phba->sli4_hba.max_cfg_param.max_eq -
- fof_vectors;
+ io_channel = phba->sli4_hba.max_cfg_param.max_eq - fof_vectors;
}
- /* The actual number of FCP event queues adopted */
- phba->cfg_fcp_io_channel = cfg_fcp_io_channel;
+ /* The actual number of FCP / NVME event queues adopted */
+ if (io_channel != phba->io_channel_irqs)
+ phba->io_channel_irqs = io_channel;
+ if (phba->cfg_fcp_io_channel > io_channel)
+ phba->cfg_fcp_io_channel = io_channel;
+ if (phba->cfg_nvme_io_channel > io_channel)
+ phba->cfg_nvme_io_channel = io_channel;
+ if (phba->cfg_nvme_io_channel < phba->cfg_nvmet_mrq)
+ phba->cfg_nvmet_mrq = phba->cfg_nvme_io_channel;
+
+ lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+ "2574 IO channels: irqs %d fcp %d nvme %d MRQ: %d\n",
+ phba->io_channel_irqs, phba->cfg_fcp_io_channel,
+ phba->cfg_nvme_io_channel, phba->cfg_nvmet_mrq);
/* Get EQ depth from module parameter, fake the default for now */
phba->sli4_hba.eq_esize = LPFC_EQE_SIZE_4B;
/* Get CQ depth from module parameter, fake the default for now */
phba->sli4_hba.cq_esize = LPFC_CQE_SIZE;
phba->sli4_hba.cq_ecount = LPFC_CQE_DEF_COUNT;
-
return 0;
-out_error:
- return -ENOMEM;
}
-/**
- * lpfc_sli4_queue_create - Create all the SLI4 queues
- * @phba: pointer to lpfc hba data structure.
- *
- * This routine is invoked to allocate all the SLI4 queues for the FCoE HBA
- * operation. For each SLI4 queue type, the parameters such as queue entry
- * count (queue depth) shall be taken from the module parameter. For now,
- * we just use some constant number as place holder.
- *
- * Return codes
- * 0 - successful
- * -ENOMEM - No availble memory
- * -EIO - The mailbox failed to complete successfully.
- **/
-int
-lpfc_sli4_queue_create(struct lpfc_hba *phba)
+static int
+lpfc_alloc_nvme_wq_cq(struct lpfc_hba *phba, int wqidx)
{
struct lpfc_queue *qdesc;
- uint32_t wqesize;
- int idx;
+ int cnt;
- /*
- * Create HBA Record arrays.
- */
- if (!phba->cfg_fcp_io_channel)
+ qdesc = lpfc_sli4_queue_alloc(phba, phba->sli4_hba.cq_esize,
+ phba->sli4_hba.cq_ecount);
+ if (!qdesc) {
+ lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+ "0508 Failed allocate fast-path NVME CQ (%d)\n",
+ wqidx);
+ return 1;
+ }
+ phba->sli4_hba.nvme_cq[wqidx] = qdesc;
+
+ cnt = LPFC_NVME_WQSIZE;
+ qdesc = lpfc_sli4_queue_alloc(phba, LPFC_WQE128_SIZE, cnt);
+ if (!qdesc) {
+ lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+ "0509 Failed allocate fast-path NVME WQ (%d)\n",
+ wqidx);
+ return 1;
+ }
+ phba->sli4_hba.nvme_wq[wqidx] = qdesc;
+ list_add_tail(&qdesc->wq_list, &phba->sli4_hba.lpfc_wq_list);
+ return 0;
+}
+
+static int
+lpfc_alloc_fcp_wq_cq(struct lpfc_hba *phba, int wqidx)
+{
+ struct lpfc_queue *qdesc;
+ uint32_t wqesize;
+
+ /* Create Fast Path FCP CQs */
+ qdesc = lpfc_sli4_queue_alloc(phba, phba->sli4_hba.cq_esize,
+ phba->sli4_hba.cq_ecount);
+ if (!qdesc) {
+ lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+ "0499 Failed allocate fast-path FCP CQ (%d)\n", wqidx);
+ return 1;
+ }
+ phba->sli4_hba.fcp_cq[wqidx] = qdesc;
+
+ /* Create Fast Path FCP WQs */
+ wqesize = (phba->fcp_embed_io) ?
+ LPFC_WQE128_SIZE : phba->sli4_hba.wq_esize;
+ qdesc = lpfc_sli4_queue_alloc(phba, wqesize, phba->sli4_hba.wq_ecount);
+ if (!qdesc) {
+ lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+ "0503 Failed allocate fast-path FCP WQ (%d)\n",
+ wqidx);
+ return 1;
+ }
+ phba->sli4_hba.fcp_wq[wqidx] = qdesc;
+ list_add_tail(&qdesc->wq_list, &phba->sli4_hba.lpfc_wq_list);
+ return 0;
+}
+
+/**
+ * lpfc_sli4_queue_create - Create all the SLI4 queues
+ * @phba: pointer to lpfc hba data structure.
+ *
+ * This routine is invoked to allocate all the SLI4 queues for the FCoE HBA
+ * operation. For each SLI4 queue type, the parameters such as queue entry
+ * count (queue depth) shall be taken from the module parameter. For now,
+ * we just use some constant number as place holder.
+ *
+ * Return codes
+ * 0 - successful
+ * -ENOMEM - No availble memory
+ * -EIO - The mailbox failed to complete successfully.
+ **/
+int
+lpfc_sli4_queue_create(struct lpfc_hba *phba)
+{
+ struct lpfc_queue *qdesc;
+ int idx, io_channel, max;
+
+ /*
+ * Create HBA Record arrays.
+ * Both NVME and FCP will share that same vectors / EQs
+ */
+ io_channel = phba->io_channel_irqs;
+ if (!io_channel)
return -ERANGE;
phba->sli4_hba.mq_esize = LPFC_MQE_SIZE;
phba->sli4_hba.wq_ecount = LPFC_WQE_DEF_COUNT;
phba->sli4_hba.rq_esize = LPFC_RQE_SIZE;
phba->sli4_hba.rq_ecount = LPFC_RQE_DEF_COUNT;
+ phba->sli4_hba.eq_esize = LPFC_EQE_SIZE_4B;
+ phba->sli4_hba.eq_ecount = LPFC_EQE_DEF_COUNT;
+ phba->sli4_hba.cq_esize = LPFC_CQE_SIZE;
+ phba->sli4_hba.cq_ecount = LPFC_CQE_DEF_COUNT;
- phba->sli4_hba.hba_eq = kzalloc((sizeof(struct lpfc_queue *) *
- phba->cfg_fcp_io_channel), GFP_KERNEL);
+ phba->sli4_hba.hba_eq = kcalloc(io_channel,
+ sizeof(struct lpfc_queue *),
+ GFP_KERNEL);
if (!phba->sli4_hba.hba_eq) {
lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
"2576 Failed allocate memory for "
goto out_error;
}
- phba->sli4_hba.fcp_cq = kzalloc((sizeof(struct lpfc_queue *) *
- phba->cfg_fcp_io_channel), GFP_KERNEL);
- if (!phba->sli4_hba.fcp_cq) {
- lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
- "2577 Failed allocate memory for fast-path "
- "CQ record array\n");
- goto out_error;
+ if (phba->cfg_fcp_io_channel) {
+ phba->sli4_hba.fcp_cq = kcalloc(phba->cfg_fcp_io_channel,
+ sizeof(struct lpfc_queue *),
+ GFP_KERNEL);
+ if (!phba->sli4_hba.fcp_cq) {
+ lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+ "2577 Failed allocate memory for "
+ "fast-path CQ record array\n");
+ goto out_error;
+ }
+ phba->sli4_hba.fcp_wq = kcalloc(phba->cfg_fcp_io_channel,
+ sizeof(struct lpfc_queue *),
+ GFP_KERNEL);
+ if (!phba->sli4_hba.fcp_wq) {
+ lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+ "2578 Failed allocate memory for "
+ "fast-path FCP WQ record array\n");
+ goto out_error;
+ }
+ /*
+ * Since the first EQ can have multiple CQs associated with it,
+ * this array is used to quickly see if we have a FCP fast-path
+ * CQ match.
+ */
+ phba->sli4_hba.fcp_cq_map = kcalloc(phba->cfg_fcp_io_channel,
+ sizeof(uint16_t),
+ GFP_KERNEL);
+ if (!phba->sli4_hba.fcp_cq_map) {
+ lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+ "2545 Failed allocate memory for "
+ "fast-path CQ map\n");
+ goto out_error;
+ }
}
- phba->sli4_hba.fcp_wq = kzalloc((sizeof(struct lpfc_queue *) *
- phba->cfg_fcp_io_channel), GFP_KERNEL);
- if (!phba->sli4_hba.fcp_wq) {
- lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
- "2578 Failed allocate memory for fast-path "
- "WQ record array\n");
- goto out_error;
- }
+ if (phba->cfg_nvme_io_channel) {
+ phba->sli4_hba.nvme_cq = kcalloc(phba->cfg_nvme_io_channel,
+ sizeof(struct lpfc_queue *),
+ GFP_KERNEL);
+ if (!phba->sli4_hba.nvme_cq) {
+ lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+ "6077 Failed allocate memory for "
+ "fast-path CQ record array\n");
+ goto out_error;
+ }
- /*
- * Since the first EQ can have multiple CQs associated with it,
- * this array is used to quickly see if we have a FCP fast-path
- * CQ match.
- */
- phba->sli4_hba.fcp_cq_map = kzalloc((sizeof(uint16_t) *
- phba->cfg_fcp_io_channel), GFP_KERNEL);
- if (!phba->sli4_hba.fcp_cq_map) {
- lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
- "2545 Failed allocate memory for fast-path "
- "CQ map\n");
- goto out_error;
+ phba->sli4_hba.nvme_wq = kcalloc(phba->cfg_nvme_io_channel,
+ sizeof(struct lpfc_queue *),
+ GFP_KERNEL);
+ if (!phba->sli4_hba.nvme_wq) {
+ lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+ "2581 Failed allocate memory for "
+ "fast-path NVME WQ record array\n");
+ goto out_error;
+ }
+
+ /*
+ * Since the first EQ can have multiple CQs associated with it,
+ * this array is used to quickly see if we have a NVME fast-path
+ * CQ match.
+ */
+ phba->sli4_hba.nvme_cq_map = kcalloc(phba->cfg_nvme_io_channel,
+ sizeof(uint16_t),
+ GFP_KERNEL);
+ if (!phba->sli4_hba.nvme_cq_map) {
+ lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+ "6078 Failed allocate memory for "
+ "fast-path CQ map\n");
+ goto out_error;
+ }
+
+ if (phba->nvmet_support) {
+ phba->sli4_hba.nvmet_cqset = kcalloc(
+ phba->cfg_nvmet_mrq,
+ sizeof(struct lpfc_queue *),
+ GFP_KERNEL);
+ if (!phba->sli4_hba.nvmet_cqset) {
+ lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+ "3121 Fail allocate memory for "
+ "fast-path CQ set array\n");
+ goto out_error;
+ }
+ phba->sli4_hba.nvmet_mrq_hdr = kcalloc(
+ phba->cfg_nvmet_mrq,
+ sizeof(struct lpfc_queue *),
+ GFP_KERNEL);
+ if (!phba->sli4_hba.nvmet_mrq_hdr) {
+ lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+ "3122 Fail allocate memory for "
+ "fast-path RQ set hdr array\n");
+ goto out_error;
+ }
+ phba->sli4_hba.nvmet_mrq_data = kcalloc(
+ phba->cfg_nvmet_mrq,
+ sizeof(struct lpfc_queue *),
+ GFP_KERNEL);
+ if (!phba->sli4_hba.nvmet_mrq_data) {
+ lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+ "3124 Fail allocate memory for "
+ "fast-path RQ set data array\n");
+ goto out_error;
+ }
+ }
}
- /*
- * Create HBA Event Queues (EQs). The cfg_fcp_io_channel specifies
- * how many EQs to create.
- */
- for (idx = 0; idx < phba->cfg_fcp_io_channel; idx++) {
+ INIT_LIST_HEAD(&phba->sli4_hba.lpfc_wq_list);
+ /* Create HBA Event Queues (EQs) */
+ for (idx = 0; idx < io_channel; idx++) {
/* Create EQs */
qdesc = lpfc_sli4_queue_alloc(phba, phba->sli4_hba.eq_esize,
phba->sli4_hba.eq_ecount);
goto out_error;
}
phba->sli4_hba.hba_eq[idx] = qdesc;
+ }
- /* Create Fast Path FCP CQs */
- qdesc = lpfc_sli4_queue_alloc(phba, phba->sli4_hba.cq_esize,
- phba->sli4_hba.cq_ecount);
- if (!qdesc) {
- lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
- "0499 Failed allocate fast-path FCP "
- "CQ (%d)\n", idx);
+ /* FCP and NVME io channels are not required to be balanced */
+
+ for (idx = 0; idx < phba->cfg_fcp_io_channel; idx++)
+ if (lpfc_alloc_fcp_wq_cq(phba, idx))
goto out_error;
- }
- phba->sli4_hba.fcp_cq[idx] = qdesc;
- /* Create Fast Path FCP WQs */
- wqesize = (phba->fcp_embed_io) ?
- LPFC_WQE128_SIZE : phba->sli4_hba.wq_esize;
- qdesc = lpfc_sli4_queue_alloc(phba, wqesize,
- phba->sli4_hba.wq_ecount);
- if (!qdesc) {
- lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
- "0503 Failed allocate fast-path FCP "
- "WQ (%d)\n", idx);
+ for (idx = 0; idx < phba->cfg_nvme_io_channel; idx++)
+ if (lpfc_alloc_nvme_wq_cq(phba, idx))
+ goto out_error;
+
+ /* allocate MRQ CQs */
+ max = phba->cfg_nvme_io_channel;
+ if (max < phba->cfg_nvmet_mrq)
+ max = phba->cfg_nvmet_mrq;
+
+ for (idx = 0; idx < max; idx++)
+ if (lpfc_alloc_nvme_wq_cq(phba, idx))
goto out_error;
+
+ if (phba->nvmet_support) {
+ for (idx = 0; idx < phba->cfg_nvmet_mrq; idx++) {
+ qdesc = lpfc_sli4_queue_alloc(phba,
+ phba->sli4_hba.cq_esize,
+ phba->sli4_hba.cq_ecount);
+ if (!qdesc) {
+ lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+ "3142 Failed allocate NVME "
+ "CQ Set (%d)\n", idx);
+ goto out_error;
+ }
+ phba->sli4_hba.nvmet_cqset[idx] = qdesc;
}
- phba->sli4_hba.fcp_wq[idx] = qdesc;
}
-
/*
* Create Slow Path Completion Queues (CQs)
*/
goto out_error;
}
phba->sli4_hba.els_wq = qdesc;
+ list_add_tail(&qdesc->wq_list, &phba->sli4_hba.lpfc_wq_list);
+
+ if (phba->cfg_enable_fc4_type & LPFC_ENABLE_NVME) {
+ /* Create NVME LS Complete Queue */
+ qdesc = lpfc_sli4_queue_alloc(phba, phba->sli4_hba.cq_esize,
+ phba->sli4_hba.cq_ecount);
+ if (!qdesc) {
+ lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+ "6079 Failed allocate NVME LS CQ\n");
+ goto out_error;
+ }
+ phba->sli4_hba.nvmels_cq = qdesc;
+
+ /* Create NVME LS Work Queue */
+ qdesc = lpfc_sli4_queue_alloc(phba, phba->sli4_hba.wq_esize,
+ phba->sli4_hba.wq_ecount);
+ if (!qdesc) {
+ lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+ "6080 Failed allocate NVME LS WQ\n");
+ goto out_error;
+ }
+ phba->sli4_hba.nvmels_wq = qdesc;
+ list_add_tail(&qdesc->wq_list, &phba->sli4_hba.lpfc_wq_list);
+ }
/*
* Create Receive Queue (RQ)
}
phba->sli4_hba.dat_rq = qdesc;
+ if (phba->nvmet_support) {
+ for (idx = 0; idx < phba->cfg_nvmet_mrq; idx++) {
+ /* Create NVMET Receive Queue for header */
+ qdesc = lpfc_sli4_queue_alloc(phba,
+ phba->sli4_hba.rq_esize,
+ phba->sli4_hba.rq_ecount);
+ if (!qdesc) {
+ lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+ "3146 Failed allocate "
+ "receive HRQ\n");
+ goto out_error;
+ }
+ phba->sli4_hba.nvmet_mrq_hdr[idx] = qdesc;
+
+ /* Only needed for header of RQ pair */
+ qdesc->rqbp = kzalloc(sizeof(struct lpfc_rqb),
+ GFP_KERNEL);
+ if (qdesc->rqbp == NULL) {
+ lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+ "6131 Failed allocate "
+ "Header RQBP\n");
+ goto out_error;
+ }
+
+ /* Create NVMET Receive Queue for data */
+ qdesc = lpfc_sli4_queue_alloc(phba,
+ phba->sli4_hba.rq_esize,
+ phba->sli4_hba.rq_ecount);
+ if (!qdesc) {
+ lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+ "3156 Failed allocate "
+ "receive DRQ\n");
+ goto out_error;
+ }
+ phba->sli4_hba.nvmet_mrq_data[idx] = qdesc;
+ }
+ }
+
/* Create the Queues needed for Flash Optimized Fabric operations */
if (phba->cfg_fof)
lpfc_fof_queue_create(phba);
return -ENOMEM;
}
+static inline void
+__lpfc_sli4_release_queue(struct lpfc_queue **qp)
+{
+ if (*qp != NULL) {
+ lpfc_sli4_queue_free(*qp);
+ *qp = NULL;
+ }
+}
+
+static inline void
+lpfc_sli4_release_queues(struct lpfc_queue ***qs, int max)
+{
+ int idx;
+
+ if (*qs == NULL)
+ return;
+
+ for (idx = 0; idx < max; idx++)
+ __lpfc_sli4_release_queue(&(*qs)[idx]);
+
+ kfree(*qs);
+ *qs = NULL;
+}
+
+static inline void
+lpfc_sli4_release_queue_map(uint16_t **qmap)
+{
+ if (*qmap != NULL) {
+ kfree(*qmap);
+ *qmap = NULL;
+ }
+}
+
/**
* lpfc_sli4_queue_destroy - Destroy all the SLI4 queues
* @phba: pointer to lpfc hba data structure.
void
lpfc_sli4_queue_destroy(struct lpfc_hba *phba)
{
- int idx;
-
if (phba->cfg_fof)
lpfc_fof_queue_destroy(phba);
- if (phba->sli4_hba.hba_eq != NULL) {
- /* Release HBA event queue */
- for (idx = 0; idx < phba->cfg_fcp_io_channel; idx++) {
- if (phba->sli4_hba.hba_eq[idx] != NULL) {
- lpfc_sli4_queue_free(
- phba->sli4_hba.hba_eq[idx]);
- phba->sli4_hba.hba_eq[idx] = NULL;
- }
- }
- kfree(phba->sli4_hba.hba_eq);
- phba->sli4_hba.hba_eq = NULL;
- }
+ /* Release HBA eqs */
+ lpfc_sli4_release_queues(&phba->sli4_hba.hba_eq, phba->io_channel_irqs);
- if (phba->sli4_hba.fcp_cq != NULL) {
- /* Release FCP completion queue */
- for (idx = 0; idx < phba->cfg_fcp_io_channel; idx++) {
- if (phba->sli4_hba.fcp_cq[idx] != NULL) {
- lpfc_sli4_queue_free(
- phba->sli4_hba.fcp_cq[idx]);
- phba->sli4_hba.fcp_cq[idx] = NULL;
- }
- }
- kfree(phba->sli4_hba.fcp_cq);
- phba->sli4_hba.fcp_cq = NULL;
- }
+ /* Release FCP cqs */
+ lpfc_sli4_release_queues(&phba->sli4_hba.fcp_cq,
+ phba->cfg_fcp_io_channel);
- if (phba->sli4_hba.fcp_wq != NULL) {
- /* Release FCP work queue */
- for (idx = 0; idx < phba->cfg_fcp_io_channel; idx++) {
- if (phba->sli4_hba.fcp_wq[idx] != NULL) {
- lpfc_sli4_queue_free(
- phba->sli4_hba.fcp_wq[idx]);
- phba->sli4_hba.fcp_wq[idx] = NULL;
- }
- }
- kfree(phba->sli4_hba.fcp_wq);
- phba->sli4_hba.fcp_wq = NULL;
- }
+ /* Release FCP wqs */
+ lpfc_sli4_release_queues(&phba->sli4_hba.fcp_wq,
+ phba->cfg_fcp_io_channel);
/* Release FCP CQ mapping array */
- if (phba->sli4_hba.fcp_cq_map != NULL) {
- kfree(phba->sli4_hba.fcp_cq_map);
- phba->sli4_hba.fcp_cq_map = NULL;
- }
+ lpfc_sli4_release_queue_map(&phba->sli4_hba.fcp_cq_map);
+
+ /* Release NVME cqs */
+ lpfc_sli4_release_queues(&phba->sli4_hba.nvme_cq,
+ phba->cfg_nvme_io_channel);
+
+ /* Release NVME wqs */
+ lpfc_sli4_release_queues(&phba->sli4_hba.nvme_wq,
+ phba->cfg_nvme_io_channel);
+
+ /* Release NVME CQ mapping array */
+ lpfc_sli4_release_queue_map(&phba->sli4_hba.nvme_cq_map);
+
+ lpfc_sli4_release_queues(&phba->sli4_hba.nvmet_cqset,
+ phba->cfg_nvmet_mrq);
+
+ lpfc_sli4_release_queues(&phba->sli4_hba.nvmet_mrq_hdr,
+ phba->cfg_nvmet_mrq);
+ lpfc_sli4_release_queues(&phba->sli4_hba.nvmet_mrq_data,
+ phba->cfg_nvmet_mrq);
/* Release mailbox command work queue */
- if (phba->sli4_hba.mbx_wq != NULL) {
- lpfc_sli4_queue_free(phba->sli4_hba.mbx_wq);
- phba->sli4_hba.mbx_wq = NULL;
- }
+ __lpfc_sli4_release_queue(&phba->sli4_hba.mbx_wq);
/* Release ELS work queue */
- if (phba->sli4_hba.els_wq != NULL) {
- lpfc_sli4_queue_free(phba->sli4_hba.els_wq);
- phba->sli4_hba.els_wq = NULL;
- }
+ __lpfc_sli4_release_queue(&phba->sli4_hba.els_wq);
+
+ /* Release ELS work queue */
+ __lpfc_sli4_release_queue(&phba->sli4_hba.nvmels_wq);
/* Release unsolicited receive queue */
- if (phba->sli4_hba.hdr_rq != NULL) {
- lpfc_sli4_queue_free(phba->sli4_hba.hdr_rq);
- phba->sli4_hba.hdr_rq = NULL;
+ __lpfc_sli4_release_queue(&phba->sli4_hba.hdr_rq);
+ __lpfc_sli4_release_queue(&phba->sli4_hba.dat_rq);
+
+ /* Release ELS complete queue */
+ __lpfc_sli4_release_queue(&phba->sli4_hba.els_cq);
+
+ /* Release NVME LS complete queue */
+ __lpfc_sli4_release_queue(&phba->sli4_hba.nvmels_cq);
+
+ /* Release mailbox command complete queue */
+ __lpfc_sli4_release_queue(&phba->sli4_hba.mbx_cq);
+
+ /* Everything on this list has been freed */
+ INIT_LIST_HEAD(&phba->sli4_hba.lpfc_wq_list);
+}
+
+int
+lpfc_post_rq_buffer(struct lpfc_hba *phba, struct lpfc_queue *hrq,
+ struct lpfc_queue *drq, int count)
+{
+ int rc, i;
+ struct lpfc_rqe hrqe;
+ struct lpfc_rqe drqe;
+ struct lpfc_rqb *rqbp;
+ struct rqb_dmabuf *rqb_buffer;
+ LIST_HEAD(rqb_buf_list);
+
+ rqbp = hrq->rqbp;
+ for (i = 0; i < count; i++) {
+ rqb_buffer = (rqbp->rqb_alloc_buffer)(phba);
+ if (!rqb_buffer)
+ break;
+ rqb_buffer->hrq = hrq;
+ rqb_buffer->drq = drq;
+ list_add_tail(&rqb_buffer->hbuf.list, &rqb_buf_list);
+ }
+ while (!list_empty(&rqb_buf_list)) {
+ list_remove_head(&rqb_buf_list, rqb_buffer, struct rqb_dmabuf,
+ hbuf.list);
+
+ hrqe.address_lo = putPaddrLow(rqb_buffer->hbuf.phys);
+ hrqe.address_hi = putPaddrHigh(rqb_buffer->hbuf.phys);
+ drqe.address_lo = putPaddrLow(rqb_buffer->dbuf.phys);
+ drqe.address_hi = putPaddrHigh(rqb_buffer->dbuf.phys);
+ rc = lpfc_sli4_rq_put(hrq, drq, &hrqe, &drqe);
+ if (rc < 0) {
+ (rqbp->rqb_free_buffer)(phba, rqb_buffer);
+ } else {
+ list_add_tail(&rqb_buffer->hbuf.list,
+ &rqbp->rqb_buffer_list);
+ rqbp->buffer_count++;
+ }
+ }
+ return 1;
+}
+
+int
+lpfc_free_rq_buffer(struct lpfc_hba *phba, struct lpfc_queue *rq)
+{
+ struct lpfc_rqb *rqbp;
+ struct lpfc_dmabuf *h_buf;
+ struct rqb_dmabuf *rqb_buffer;
+
+ rqbp = rq->rqbp;
+ while (!list_empty(&rqbp->rqb_buffer_list)) {
+ list_remove_head(&rqbp->rqb_buffer_list, h_buf,
+ struct lpfc_dmabuf, list);
+
+ rqb_buffer = container_of(h_buf, struct rqb_dmabuf, hbuf);
+ (rqbp->rqb_free_buffer)(phba, rqb_buffer);
+ rqbp->buffer_count--;
}
- if (phba->sli4_hba.dat_rq != NULL) {
- lpfc_sli4_queue_free(phba->sli4_hba.dat_rq);
- phba->sli4_hba.dat_rq = NULL;
+ return 1;
+}
+
+static int
+lpfc_create_wq_cq(struct lpfc_hba *phba, struct lpfc_queue *eq,
+ struct lpfc_queue *cq, struct lpfc_queue *wq, uint16_t *cq_map,
+ int qidx, uint32_t qtype)
+{
+ struct lpfc_sli_ring *pring;
+ int rc;
+
+ if (!eq || !cq || !wq) {
+ lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+ "6085 Fast-path %s (%d) not allocated\n",
+ ((eq) ? ((cq) ? "WQ" : "CQ") : "EQ"), qidx);
+ return -ENOMEM;
}
- /* Release ELS complete queue */
- if (phba->sli4_hba.els_cq != NULL) {
- lpfc_sli4_queue_free(phba->sli4_hba.els_cq);
- phba->sli4_hba.els_cq = NULL;
+ /* create the Cq first */
+ rc = lpfc_cq_create(phba, cq, eq,
+ (qtype == LPFC_MBOX) ? LPFC_MCQ : LPFC_WCQ, qtype);
+ if (rc) {
+ lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+ "6086 Failed setup of CQ (%d), rc = 0x%x\n",
+ qidx, (uint32_t)rc);
+ return rc;
}
- /* Release mailbox command complete queue */
- if (phba->sli4_hba.mbx_cq != NULL) {
- lpfc_sli4_queue_free(phba->sli4_hba.mbx_cq);
- phba->sli4_hba.mbx_cq = NULL;
+ if (qtype != LPFC_MBOX) {
+ /* Setup nvme_cq_map for fast lookup */
+ if (cq_map)
+ *cq_map = cq->queue_id;
+
+ lpfc_printf_log(phba, KERN_INFO, LOG_INIT,
+ "6087 CQ setup: cq[%d]-id=%d, parent eq[%d]-id=%d\n",
+ qidx, cq->queue_id, qidx, eq->queue_id);
+
+ /* create the wq */
+ rc = lpfc_wq_create(phba, wq, cq, qtype);
+ if (rc) {
+ lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+ "6123 Fail setup fastpath WQ (%d), rc = 0x%x\n",
+ qidx, (uint32_t)rc);
+ /* no need to tear down cq - caller will do so */
+ return rc;
+ }
+
+ /* Bind this CQ/WQ to the NVME ring */
+ pring = wq->pring;
+ pring->sli.sli4.wqp = (void *)wq;
+ cq->pring = pring;
+
+ lpfc_printf_log(phba, KERN_INFO, LOG_INIT,
+ "2593 WQ setup: wq[%d]-id=%d assoc=%d, cq[%d]-id=%d\n",
+ qidx, wq->queue_id, wq->assoc_qid, qidx, cq->queue_id);
+ } else {
+ rc = lpfc_mq_create(phba, wq, cq, LPFC_MBOX);
+ if (rc) {
+ lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+ "0539 Failed setup of slow-path MQ: "
+ "rc = 0x%x\n", rc);
+ /* no need to tear down cq - caller will do so */
+ return rc;
+ }
+
+ lpfc_printf_log(phba, KERN_INFO, LOG_INIT,
+ "2589 MBX MQ setup: wq-id=%d, parent cq-id=%d\n",
+ phba->sli4_hba.mbx_wq->queue_id,
+ phba->sli4_hba.mbx_cq->queue_id);
}
- return;
+ return 0;
}
/**
int
lpfc_sli4_queue_setup(struct lpfc_hba *phba)
{
- struct lpfc_sli *psli = &phba->sli;
- struct lpfc_sli_ring *pring;
- int rc = -ENOMEM;
- int fcp_eqidx, fcp_cqidx, fcp_wqidx;
- int fcp_cq_index = 0;
uint32_t shdr_status, shdr_add_status;
union lpfc_sli4_cfg_shdr *shdr;
LPFC_MBOXQ_t *mboxq;
- uint32_t length;
+ int qidx;
+ uint32_t length, io_channel;
+ int rc = -ENOMEM;
/* Check for dual-ULP support */
mboxq = (LPFC_MBOXQ_t *)mempool_alloc(phba->mbox_mem_pool, GFP_KERNEL);
/*
* Set up HBA Event Queues (EQs)
*/
+ io_channel = phba->io_channel_irqs;
/* Set up HBA event queue */
- if (phba->cfg_fcp_io_channel && !phba->sli4_hba.hba_eq) {
+ if (io_channel && !phba->sli4_hba.hba_eq) {
lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
"3147 Fast-path EQs not allocated\n");
rc = -ENOMEM;
goto out_error;
}
- for (fcp_eqidx = 0; fcp_eqidx < phba->cfg_fcp_io_channel; fcp_eqidx++) {
- if (!phba->sli4_hba.hba_eq[fcp_eqidx]) {
+ for (qidx = 0; qidx < io_channel; qidx++) {
+ if (!phba->sli4_hba.hba_eq[qidx]) {
lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
"0522 Fast-path EQ (%d) not "
- "allocated\n", fcp_eqidx);
+ "allocated\n", qidx);
rc = -ENOMEM;
- goto out_destroy_hba_eq;
+ goto out_destroy;
}
- rc = lpfc_eq_create(phba, phba->sli4_hba.hba_eq[fcp_eqidx],
- (phba->cfg_fcp_imax / phba->cfg_fcp_io_channel));
+ rc = lpfc_eq_create(phba, phba->sli4_hba.hba_eq[qidx],
+ phba->cfg_fcp_imax);
if (rc) {
lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
"0523 Failed setup of fast-path EQ "
- "(%d), rc = 0x%x\n", fcp_eqidx,
+ "(%d), rc = 0x%x\n", qidx,
(uint32_t)rc);
- goto out_destroy_hba_eq;
+ goto out_destroy;
}
lpfc_printf_log(phba, KERN_INFO, LOG_INIT,
- "2584 HBA EQ setup: "
- "queue[%d]-id=%d\n", fcp_eqidx,
- phba->sli4_hba.hba_eq[fcp_eqidx]->queue_id);
+ "2584 HBA EQ setup: queue[%d]-id=%d\n",
+ qidx, phba->sli4_hba.hba_eq[qidx]->queue_id);
}
- /* Set up fast-path FCP Response Complete Queue */
- if (!phba->sli4_hba.fcp_cq) {
- lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
- "3148 Fast-path FCP CQ array not "
- "allocated\n");
- rc = -ENOMEM;
- goto out_destroy_hba_eq;
+ if (phba->cfg_nvme_io_channel) {
+ if (!phba->sli4_hba.nvme_cq || !phba->sli4_hba.nvme_wq) {
+ lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+ "6084 Fast-path NVME %s array not allocated\n",
+ (phba->sli4_hba.nvme_cq) ? "CQ" : "WQ");
+ rc = -ENOMEM;
+ goto out_destroy;
+ }
+
+ for (qidx = 0; qidx < phba->cfg_nvme_io_channel; qidx++) {
+ rc = lpfc_create_wq_cq(phba,
+ phba->sli4_hba.hba_eq[
+ qidx % io_channel],
+ phba->sli4_hba.nvme_cq[qidx],
+ phba->sli4_hba.nvme_wq[qidx],
+ &phba->sli4_hba.nvme_cq_map[qidx],
+ qidx, LPFC_NVME);
+ if (rc) {
+ lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+ "6123 Failed to setup fastpath "
+ "NVME WQ/CQ (%d), rc = 0x%x\n",
+ qidx, (uint32_t)rc);
+ goto out_destroy;
+ }
+ }
}
- for (fcp_cqidx = 0; fcp_cqidx < phba->cfg_fcp_io_channel; fcp_cqidx++) {
- if (!phba->sli4_hba.fcp_cq[fcp_cqidx]) {
+ if (phba->cfg_fcp_io_channel) {
+ /* Set up fast-path FCP Response Complete Queue */
+ if (!phba->sli4_hba.fcp_cq || !phba->sli4_hba.fcp_wq) {
lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
- "0526 Fast-path FCP CQ (%d) not "
- "allocated\n", fcp_cqidx);
+ "3148 Fast-path FCP %s array not allocated\n",
+ phba->sli4_hba.fcp_cq ? "WQ" : "CQ");
rc = -ENOMEM;
- goto out_destroy_fcp_cq;
+ goto out_destroy;
}
- rc = lpfc_cq_create(phba, phba->sli4_hba.fcp_cq[fcp_cqidx],
- phba->sli4_hba.hba_eq[fcp_cqidx], LPFC_WCQ, LPFC_FCP);
- if (rc) {
- lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
- "0527 Failed setup of fast-path FCP "
- "CQ (%d), rc = 0x%x\n", fcp_cqidx,
- (uint32_t)rc);
- goto out_destroy_fcp_cq;
+
+ for (qidx = 0; qidx < phba->cfg_fcp_io_channel; qidx++) {
+ rc = lpfc_create_wq_cq(phba,
+ phba->sli4_hba.hba_eq[
+ qidx % io_channel],
+ phba->sli4_hba.fcp_cq[qidx],
+ phba->sli4_hba.fcp_wq[qidx],
+ &phba->sli4_hba.fcp_cq_map[qidx],
+ qidx, LPFC_FCP);
+ if (rc) {
+ lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+ "0535 Failed to setup fastpath "
+ "FCP WQ/CQ (%d), rc = 0x%x\n",
+ qidx, (uint32_t)rc);
+ goto out_destroy;
+ }
}
+ }
- /* Setup fcp_cq_map for fast lookup */
- phba->sli4_hba.fcp_cq_map[fcp_cqidx] =
- phba->sli4_hba.fcp_cq[fcp_cqidx]->queue_id;
+ /*
+ * Set up Slow Path Complete Queues (CQs)
+ */
- lpfc_printf_log(phba, KERN_INFO, LOG_INIT,
- "2588 FCP CQ setup: cq[%d]-id=%d, "
- "parent seq[%d]-id=%d\n",
- fcp_cqidx,
- phba->sli4_hba.fcp_cq[fcp_cqidx]->queue_id,
- fcp_cqidx,
- phba->sli4_hba.hba_eq[fcp_cqidx]->queue_id);
- }
+ /* Set up slow-path MBOX CQ/MQ */
- /* Set up fast-path FCP Work Queue */
- if (!phba->sli4_hba.fcp_wq) {
+ if (!phba->sli4_hba.mbx_cq || !phba->sli4_hba.mbx_wq) {
lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
- "3149 Fast-path FCP WQ array not "
- "allocated\n");
+ "0528 %s not allocated\n",
+ phba->sli4_hba.mbx_cq ?
+ "Mailbox WQ" : "Mailbox CQ");
rc = -ENOMEM;
- goto out_destroy_fcp_cq;
+ goto out_destroy;
}
- for (fcp_wqidx = 0; fcp_wqidx < phba->cfg_fcp_io_channel; fcp_wqidx++) {
- if (!phba->sli4_hba.fcp_wq[fcp_wqidx]) {
+ rc = lpfc_create_wq_cq(phba, phba->sli4_hba.hba_eq[0],
+ phba->sli4_hba.mbx_cq,
+ phba->sli4_hba.mbx_wq,
+ NULL, 0, LPFC_MBOX);
+ if (rc) {
+ lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+ "0529 Failed setup of mailbox WQ/CQ: rc = 0x%x\n",
+ (uint32_t)rc);
+ goto out_destroy;
+ }
+ if (phba->nvmet_support) {
+ if (!phba->sli4_hba.nvmet_cqset) {
lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
- "0534 Fast-path FCP WQ (%d) not "
- "allocated\n", fcp_wqidx);
+ "3165 Fast-path NVME CQ Set "
+ "array not allocated\n");
rc = -ENOMEM;
- goto out_destroy_fcp_wq;
+ goto out_destroy;
}
- rc = lpfc_wq_create(phba, phba->sli4_hba.fcp_wq[fcp_wqidx],
- phba->sli4_hba.fcp_cq[fcp_wqidx],
- LPFC_FCP);
- if (rc) {
- lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
- "0535 Failed setup of fast-path FCP "
- "WQ (%d), rc = 0x%x\n", fcp_wqidx,
- (uint32_t)rc);
- goto out_destroy_fcp_wq;
+ if (phba->cfg_nvmet_mrq > 1) {
+ rc = lpfc_cq_create_set(phba,
+ phba->sli4_hba.nvmet_cqset,
+ phba->sli4_hba.hba_eq,
+ LPFC_WCQ, LPFC_NVMET);
+ if (rc) {
+ lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+ "3164 Failed setup of NVME CQ "
+ "Set, rc = 0x%x\n",
+ (uint32_t)rc);
+ goto out_destroy;
+ }
+ } else {
+ /* Set up NVMET Receive Complete Queue */
+ rc = lpfc_cq_create(phba, phba->sli4_hba.nvmet_cqset[0],
+ phba->sli4_hba.hba_eq[0],
+ LPFC_WCQ, LPFC_NVMET);
+ if (rc) {
+ lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+ "6089 Failed setup NVMET CQ: "
+ "rc = 0x%x\n", (uint32_t)rc);
+ goto out_destroy;
+ }
+ lpfc_printf_log(phba, KERN_INFO, LOG_INIT,
+ "6090 NVMET CQ setup: cq-id=%d, "
+ "parent eq-id=%d\n",
+ phba->sli4_hba.nvmet_cqset[0]->queue_id,
+ phba->sli4_hba.hba_eq[0]->queue_id);
}
-
- /* Bind this WQ to the next FCP ring */
- pring = &psli->ring[MAX_SLI3_CONFIGURED_RINGS + fcp_wqidx];
- pring->sli.sli4.wqp = (void *)phba->sli4_hba.fcp_wq[fcp_wqidx];
- phba->sli4_hba.fcp_cq[fcp_wqidx]->pring = pring;
-
- lpfc_printf_log(phba, KERN_INFO, LOG_INIT,
- "2591 FCP WQ setup: wq[%d]-id=%d, "
- "parent cq[%d]-id=%d\n",
- fcp_wqidx,
- phba->sli4_hba.fcp_wq[fcp_wqidx]->queue_id,
- fcp_cq_index,
- phba->sli4_hba.fcp_cq[fcp_wqidx]->queue_id);
}
- /*
- * Set up Complete Queues (CQs)
- */
- /* Set up slow-path MBOX Complete Queue as the first CQ */
- if (!phba->sli4_hba.mbx_cq) {
+ /* Set up slow-path ELS WQ/CQ */
+ if (!phba->sli4_hba.els_cq || !phba->sli4_hba.els_wq) {
lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
- "0528 Mailbox CQ not allocated\n");
+ "0530 ELS %s not allocated\n",
+ phba->sli4_hba.els_cq ? "WQ" : "CQ");
rc = -ENOMEM;
- goto out_destroy_fcp_wq;
+ goto out_destroy;
}
- rc = lpfc_cq_create(phba, phba->sli4_hba.mbx_cq,
- phba->sli4_hba.hba_eq[0], LPFC_MCQ, LPFC_MBOX);
+ rc = lpfc_create_wq_cq(phba, phba->sli4_hba.hba_eq[0],
+ phba->sli4_hba.els_cq,
+ phba->sli4_hba.els_wq,
+ NULL, 0, LPFC_ELS);
if (rc) {
lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
- "0529 Failed setup of slow-path mailbox CQ: "
- "rc = 0x%x\n", (uint32_t)rc);
- goto out_destroy_fcp_wq;
+ "0529 Failed setup of ELS WQ/CQ: rc = 0x%x\n",
+ (uint32_t)rc);
+ goto out_destroy;
}
lpfc_printf_log(phba, KERN_INFO, LOG_INIT,
- "2585 MBX CQ setup: cq-id=%d, parent eq-id=%d\n",
- phba->sli4_hba.mbx_cq->queue_id,
- phba->sli4_hba.hba_eq[0]->queue_id);
+ "2590 ELS WQ setup: wq-id=%d, parent cq-id=%d\n",
+ phba->sli4_hba.els_wq->queue_id,
+ phba->sli4_hba.els_cq->queue_id);
- /* Set up slow-path ELS Complete Queue */
- if (!phba->sli4_hba.els_cq) {
- lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
- "0530 ELS CQ not allocated\n");
- rc = -ENOMEM;
- goto out_destroy_mbx_cq;
- }
- rc = lpfc_cq_create(phba, phba->sli4_hba.els_cq,
- phba->sli4_hba.hba_eq[0], LPFC_WCQ, LPFC_ELS);
- if (rc) {
- lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
- "0531 Failed setup of slow-path ELS CQ: "
+ if (phba->cfg_nvme_io_channel) {
+ /* Set up NVME LS Complete Queue */
+ if (!phba->sli4_hba.nvmels_cq || !phba->sli4_hba.nvmels_wq) {
+ lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+ "6091 LS %s not allocated\n",
+ phba->sli4_hba.nvmels_cq ? "WQ" : "CQ");
+ rc = -ENOMEM;
+ goto out_destroy;
+ }
+ rc = lpfc_create_wq_cq(phba, phba->sli4_hba.hba_eq[0],
+ phba->sli4_hba.nvmels_cq,
+ phba->sli4_hba.nvmels_wq,
+ NULL, 0, LPFC_NVME_LS);
+ if (rc) {
+ lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+ "0529 Failed setup of NVVME LS WQ/CQ: "
"rc = 0x%x\n", (uint32_t)rc);
- goto out_destroy_mbx_cq;
+ goto out_destroy;
+ }
+
+ lpfc_printf_log(phba, KERN_INFO, LOG_INIT,
+ "6096 ELS WQ setup: wq-id=%d, "
+ "parent cq-id=%d\n",
+ phba->sli4_hba.nvmels_wq->queue_id,
+ phba->sli4_hba.nvmels_cq->queue_id);
}
- lpfc_printf_log(phba, KERN_INFO, LOG_INIT,
- "2586 ELS CQ setup: cq-id=%d, parent eq-id=%d\n",
- phba->sli4_hba.els_cq->queue_id,
- phba->sli4_hba.hba_eq[0]->queue_id);
/*
- * Set up all the Work Queues (WQs)
+ * Create NVMET Receive Queue (RQ)
*/
+ if (phba->nvmet_support) {
+ if ((!phba->sli4_hba.nvmet_cqset) ||
+ (!phba->sli4_hba.nvmet_mrq_hdr) ||
+ (!phba->sli4_hba.nvmet_mrq_data)) {
+ lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+ "6130 MRQ CQ Queues not "
+ "allocated\n");
+ rc = -ENOMEM;
+ goto out_destroy;
+ }
+ if (phba->cfg_nvmet_mrq > 1) {
+ rc = lpfc_mrq_create(phba,
+ phba->sli4_hba.nvmet_mrq_hdr,
+ phba->sli4_hba.nvmet_mrq_data,
+ phba->sli4_hba.nvmet_cqset,
+ LPFC_NVMET);
+ if (rc) {
+ lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+ "6098 Failed setup of NVMET "
+ "MRQ: rc = 0x%x\n",
+ (uint32_t)rc);
+ goto out_destroy;
+ }
- /* Set up Mailbox Command Queue */
- if (!phba->sli4_hba.mbx_wq) {
- lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
- "0538 Slow-path MQ not allocated\n");
- rc = -ENOMEM;
- goto out_destroy_els_cq;
- }
- rc = lpfc_mq_create(phba, phba->sli4_hba.mbx_wq,
- phba->sli4_hba.mbx_cq, LPFC_MBOX);
- if (rc) {
- lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
- "0539 Failed setup of slow-path MQ: "
- "rc = 0x%x\n", rc);
- goto out_destroy_els_cq;
- }
- lpfc_printf_log(phba, KERN_INFO, LOG_INIT,
- "2589 MBX MQ setup: wq-id=%d, parent cq-id=%d\n",
- phba->sli4_hba.mbx_wq->queue_id,
- phba->sli4_hba.mbx_cq->queue_id);
-
- /* Set up slow-path ELS Work Queue */
- if (!phba->sli4_hba.els_wq) {
- lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
- "0536 Slow-path ELS WQ not allocated\n");
- rc = -ENOMEM;
- goto out_destroy_mbx_wq;
- }
- rc = lpfc_wq_create(phba, phba->sli4_hba.els_wq,
- phba->sli4_hba.els_cq, LPFC_ELS);
- if (rc) {
- lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
- "0537 Failed setup of slow-path ELS WQ: "
- "rc = 0x%x\n", (uint32_t)rc);
- goto out_destroy_mbx_wq;
- }
+ } else {
+ rc = lpfc_rq_create(phba,
+ phba->sli4_hba.nvmet_mrq_hdr[0],
+ phba->sli4_hba.nvmet_mrq_data[0],
+ phba->sli4_hba.nvmet_cqset[0],
+ LPFC_NVMET);
+ if (rc) {
+ lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+ "6057 Failed setup of NVMET "
+ "Receive Queue: rc = 0x%x\n",
+ (uint32_t)rc);
+ goto out_destroy;
+ }
- /* Bind this WQ to the ELS ring */
- pring = &psli->ring[LPFC_ELS_RING];
- pring->sli.sli4.wqp = (void *)phba->sli4_hba.els_wq;
- phba->sli4_hba.els_cq->pring = pring;
+ lpfc_printf_log(
+ phba, KERN_INFO, LOG_INIT,
+ "6099 NVMET RQ setup: hdr-rq-id=%d, "
+ "dat-rq-id=%d parent cq-id=%d\n",
+ phba->sli4_hba.nvmet_mrq_hdr[0]->queue_id,
+ phba->sli4_hba.nvmet_mrq_data[0]->queue_id,
+ phba->sli4_hba.nvmet_cqset[0]->queue_id);
- lpfc_printf_log(phba, KERN_INFO, LOG_INIT,
- "2590 ELS WQ setup: wq-id=%d, parent cq-id=%d\n",
- phba->sli4_hba.els_wq->queue_id,
- phba->sli4_hba.els_cq->queue_id);
+ }
+ }
- /*
- * Create Receive Queue (RQ)
- */
if (!phba->sli4_hba.hdr_rq || !phba->sli4_hba.dat_rq) {
lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
"0540 Receive Queue not allocated\n");
rc = -ENOMEM;
- goto out_destroy_els_wq;
+ goto out_destroy;
}
lpfc_rq_adjust_repost(phba, phba->sli4_hba.hdr_rq, LPFC_ELS_HBQ);
lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
"0541 Failed setup of Receive Queue: "
"rc = 0x%x\n", (uint32_t)rc);
- goto out_destroy_fcp_wq;
+ goto out_destroy;
}
lpfc_printf_log(phba, KERN_INFO, LOG_INIT,
lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
"0549 Failed setup of FOF Queues: "
"rc = 0x%x\n", rc);
- goto out_destroy_els_rq;
+ goto out_destroy;
}
}
* Configure EQ delay multipier for interrupt coalescing using
* MODIFY_EQ_DELAY for all EQs created, LPFC_MAX_EQ_DELAY at a time.
*/
- for (fcp_eqidx = 0; fcp_eqidx < phba->cfg_fcp_io_channel;
- fcp_eqidx += LPFC_MAX_EQ_DELAY)
- lpfc_modify_fcp_eq_delay(phba, fcp_eqidx);
+ for (qidx = 0; qidx < io_channel; qidx += LPFC_MAX_EQ_DELAY)
+ lpfc_modify_hba_eq_delay(phba, qidx);
return 0;
-out_destroy_els_rq:
- lpfc_rq_destroy(phba, phba->sli4_hba.hdr_rq, phba->sli4_hba.dat_rq);
-out_destroy_els_wq:
- lpfc_wq_destroy(phba, phba->sli4_hba.els_wq);
-out_destroy_mbx_wq:
- lpfc_mq_destroy(phba, phba->sli4_hba.mbx_wq);
-out_destroy_els_cq:
- lpfc_cq_destroy(phba, phba->sli4_hba.els_cq);
-out_destroy_mbx_cq:
- lpfc_cq_destroy(phba, phba->sli4_hba.mbx_cq);
-out_destroy_fcp_wq:
- for (--fcp_wqidx; fcp_wqidx >= 0; fcp_wqidx--)
- lpfc_wq_destroy(phba, phba->sli4_hba.fcp_wq[fcp_wqidx]);
-out_destroy_fcp_cq:
- for (--fcp_cqidx; fcp_cqidx >= 0; fcp_cqidx--)
- lpfc_cq_destroy(phba, phba->sli4_hba.fcp_cq[fcp_cqidx]);
-out_destroy_hba_eq:
- for (--fcp_eqidx; fcp_eqidx >= 0; fcp_eqidx--)
- lpfc_eq_destroy(phba, phba->sli4_hba.hba_eq[fcp_eqidx]);
+out_destroy:
+ lpfc_sli4_queue_unset(phba);
out_error:
return rc;
}
void
lpfc_sli4_queue_unset(struct lpfc_hba *phba)
{
- int fcp_qidx;
+ int qidx;
/* Unset the queues created for Flash Optimized Fabric operations */
if (phba->cfg_fof)
lpfc_fof_queue_destroy(phba);
+
/* Unset mailbox command work queue */
- lpfc_mq_destroy(phba, phba->sli4_hba.mbx_wq);
+ if (phba->sli4_hba.mbx_wq)
+ lpfc_mq_destroy(phba, phba->sli4_hba.mbx_wq);
+
+ /* Unset NVME LS work queue */
+ if (phba->sli4_hba.nvmels_wq)
+ lpfc_wq_destroy(phba, phba->sli4_hba.nvmels_wq);
+
/* Unset ELS work queue */
- lpfc_wq_destroy(phba, phba->sli4_hba.els_wq);
+ if (phba->sli4_hba.els_cq)
+ lpfc_wq_destroy(phba, phba->sli4_hba.els_wq);
+
/* Unset unsolicited receive queue */
- lpfc_rq_destroy(phba, phba->sli4_hba.hdr_rq, phba->sli4_hba.dat_rq);
+ if (phba->sli4_hba.hdr_rq)
+ lpfc_rq_destroy(phba, phba->sli4_hba.hdr_rq,
+ phba->sli4_hba.dat_rq);
+
/* Unset FCP work queue */
- if (phba->sli4_hba.fcp_wq) {
- for (fcp_qidx = 0; fcp_qidx < phba->cfg_fcp_io_channel;
- fcp_qidx++)
- lpfc_wq_destroy(phba, phba->sli4_hba.fcp_wq[fcp_qidx]);
+ if (phba->sli4_hba.fcp_wq)
+ for (qidx = 0; qidx < phba->cfg_fcp_io_channel; qidx++)
+ lpfc_wq_destroy(phba, phba->sli4_hba.fcp_wq[qidx]);
+
+ /* Unset NVME work queue */
+ if (phba->sli4_hba.nvme_wq) {
+ for (qidx = 0; qidx < phba->cfg_nvme_io_channel; qidx++)
+ lpfc_wq_destroy(phba, phba->sli4_hba.nvme_wq[qidx]);
}
+
/* Unset mailbox command complete queue */
- lpfc_cq_destroy(phba, phba->sli4_hba.mbx_cq);
+ if (phba->sli4_hba.mbx_cq)
+ lpfc_cq_destroy(phba, phba->sli4_hba.mbx_cq);
+
/* Unset ELS complete queue */
- lpfc_cq_destroy(phba, phba->sli4_hba.els_cq);
- /* Unset FCP response complete queue */
- if (phba->sli4_hba.fcp_cq) {
- for (fcp_qidx = 0; fcp_qidx < phba->cfg_fcp_io_channel;
- fcp_qidx++)
- lpfc_cq_destroy(phba, phba->sli4_hba.fcp_cq[fcp_qidx]);
+ if (phba->sli4_hba.els_cq)
+ lpfc_cq_destroy(phba, phba->sli4_hba.els_cq);
+
+ /* Unset NVME LS complete queue */
+ if (phba->sli4_hba.nvmels_cq)
+ lpfc_cq_destroy(phba, phba->sli4_hba.nvmels_cq);
+
+ /* Unset NVME response complete queue */
+ if (phba->sli4_hba.nvme_cq)
+ for (qidx = 0; qidx < phba->cfg_nvme_io_channel; qidx++)
+ lpfc_cq_destroy(phba, phba->sli4_hba.nvme_cq[qidx]);
+
+ /* Unset NVMET MRQ queue */
+ if (phba->sli4_hba.nvmet_mrq_hdr) {
+ for (qidx = 0; qidx < phba->cfg_nvmet_mrq; qidx++)
+ lpfc_rq_destroy(phba,
+ phba->sli4_hba.nvmet_mrq_hdr[qidx],
+ phba->sli4_hba.nvmet_mrq_data[qidx]);
}
- /* Unset fast-path event queue */
- if (phba->sli4_hba.hba_eq) {
- for (fcp_qidx = 0; fcp_qidx < phba->cfg_fcp_io_channel;
- fcp_qidx++)
- lpfc_eq_destroy(phba, phba->sli4_hba.hba_eq[fcp_qidx]);
+
+ /* Unset NVMET CQ Set complete queue */
+ if (phba->sli4_hba.nvmet_cqset) {
+ for (qidx = 0; qidx < phba->cfg_nvmet_mrq; qidx++)
+ lpfc_cq_destroy(phba,
+ phba->sli4_hba.nvmet_cqset[qidx]);
}
+
+ /* Unset FCP response complete queue */
+ if (phba->sli4_hba.fcp_cq)
+ for (qidx = 0; qidx < phba->cfg_fcp_io_channel; qidx++)
+ lpfc_cq_destroy(phba, phba->sli4_hba.fcp_cq[qidx]);
+
+ /* Unset fast-path event queue */
+ if (phba->sli4_hba.hba_eq)
+ for (qidx = 0; qidx < phba->io_channel_irqs; qidx++)
+ lpfc_eq_destroy(phba, phba->sli4_hba.hba_eq[qidx]);
}
/**
* @phba: pointer to lpfc hba data structure.
*
* This routine is invoked to enable the MSI-X interrupt vectors to device
- * with SLI-3 interface specs. The kernel function pci_enable_msix_exact()
- * is called to enable the MSI-X vectors. Note that pci_enable_msix_exact(),
- * once invoked, enables either all or nothing, depending on the current
- * availability of PCI vector resources. The device driver is responsible
- * for calling the individual request_irq() to register each MSI-X vector
- * with a interrupt handler, which is done in this function. Note that
- * later when device is unloading, the driver should always call free_irq()
- * on all MSI-X vectors it has done request_irq() on before calling
- * pci_disable_msix(). Failure to do so results in a BUG_ON() and a device
- * will be left with MSI-X enabled and leaks its vectors.
+ * with SLI-3 interface specs.
*
* Return codes
* 0 - successful
static int
lpfc_sli_enable_msix(struct lpfc_hba *phba)
{
- int rc, i;
+ int rc;
LPFC_MBOXQ_t *pmb;
/* Set up MSI-X multi-message vectors */
- for (i = 0; i < LPFC_MSIX_VECTORS; i++)
- phba->msix_entries[i].entry = i;
-
- /* Configure MSI-X capability structure */
- rc = pci_enable_msix_exact(phba->pcidev, phba->msix_entries,
- LPFC_MSIX_VECTORS);
- if (rc) {
+ rc = pci_alloc_irq_vectors(phba->pcidev,
+ LPFC_MSIX_VECTORS, LPFC_MSIX_VECTORS, PCI_IRQ_MSIX);
+ if (rc < 0) {
lpfc_printf_log(phba, KERN_INFO, LOG_INIT,
"0420 PCI enable MSI-X failed (%d)\n", rc);
goto vec_fail_out;
}
- for (i = 0; i < LPFC_MSIX_VECTORS; i++)
- lpfc_printf_log(phba, KERN_INFO, LOG_INIT,
- "0477 MSI-X entry[%d]: vector=x%x "
- "message=%d\n", i,
- phba->msix_entries[i].vector,
- phba->msix_entries[i].entry);
+
/*
* Assign MSI-X vectors to interrupt handlers
*/
/* vector-0 is associated to slow-path handler */
- rc = request_irq(phba->msix_entries[0].vector,
+ rc = request_irq(pci_irq_vector(phba->pcidev, 0),
&lpfc_sli_sp_intr_handler, 0,
LPFC_SP_DRIVER_HANDLER_NAME, phba);
if (rc) {
}
/* vector-1 is associated to fast-path handler */
- rc = request_irq(phba->msix_entries[1].vector,
+ rc = request_irq(pci_irq_vector(phba->pcidev, 1),
&lpfc_sli_fp_intr_handler, 0,
LPFC_FP_DRIVER_HANDLER_NAME, phba);
mem_fail_out:
/* free the irq already requested */
- free_irq(phba->msix_entries[1].vector, phba);
+ free_irq(pci_irq_vector(phba->pcidev, 1), phba);
irq_fail_out:
/* free the irq already requested */
- free_irq(phba->msix_entries[0].vector, phba);
+ free_irq(pci_irq_vector(phba->pcidev, 0), phba);
msi_fail_out:
/* Unconfigure MSI-X capability structure */
- pci_disable_msix(phba->pcidev);
+ pci_free_irq_vectors(phba->pcidev);
vec_fail_out:
return rc;
}
-/**
- * lpfc_sli_disable_msix - Disable MSI-X interrupt mode on SLI-3 device.
- * @phba: pointer to lpfc hba data structure.
- *
- * This routine is invoked to release the MSI-X vectors and then disable the
- * MSI-X interrupt mode to device with SLI-3 interface spec.
- **/
-static void
-lpfc_sli_disable_msix(struct lpfc_hba *phba)
-{
- int i;
-
- /* Free up MSI-X multi-message vectors */
- for (i = 0; i < LPFC_MSIX_VECTORS; i++)
- free_irq(phba->msix_entries[i].vector, phba);
- /* Disable MSI-X */
- pci_disable_msix(phba->pcidev);
-
- return;
-}
-
/**
* lpfc_sli_enable_msi - Enable MSI interrupt mode on SLI-3 device.
* @phba: pointer to lpfc hba data structure.
return rc;
}
-/**
- * lpfc_sli_disable_msi - Disable MSI interrupt mode to SLI-3 device.
- * @phba: pointer to lpfc hba data structure.
- *
- * This routine is invoked to disable the MSI interrupt mode to device with
- * SLI-3 interface spec. The driver calls free_irq() on MSI vector it has
- * done request_irq() on before calling pci_disable_msi(). Failure to do so
- * results in a BUG_ON() and a device will be left with MSI enabled and leaks
- * its vector.
- */
-static void
-lpfc_sli_disable_msi(struct lpfc_hba *phba)
-{
- free_irq(phba->pcidev->irq, phba);
- pci_disable_msi(phba->pcidev);
- return;
-}
-
/**
* lpfc_sli_enable_intr - Enable device interrupt to SLI-3 device.
* @phba: pointer to lpfc hba data structure.
static void
lpfc_sli_disable_intr(struct lpfc_hba *phba)
{
- /* Disable the currently initialized interrupt mode */
+ int nr_irqs, i;
+
if (phba->intr_type == MSIX)
- lpfc_sli_disable_msix(phba);
- else if (phba->intr_type == MSI)
- lpfc_sli_disable_msi(phba);
- else if (phba->intr_type == INTx)
- free_irq(phba->pcidev->irq, phba);
+ nr_irqs = LPFC_MSIX_VECTORS;
+ else
+ nr_irqs = 1;
+
+ for (i = 0; i < nr_irqs; i++)
+ free_irq(pci_irq_vector(phba->pcidev, i), phba);
+ pci_free_irq_vectors(phba->pcidev);
/* Reset interrupt management states */
phba->intr_type = NONE;
phba->sli.slistat.sli_intr = 0;
-
- return;
}
/**
- * lpfc_find_next_cpu - Find next available CPU that matches the phys_id
+ * lpfc_cpu_affinity_check - Check vector CPU affinity mappings
* @phba: pointer to lpfc hba data structure.
+ * @vectors: number of msix vectors allocated.
*
- * Find next available CPU to use for IRQ to CPU affinity.
+ * The routine will figure out the CPU affinity assignment for every
+ * MSI-X vector allocated for the HBA. The hba_eq_hdl will be updated
+ * with a pointer to the CPU mask that defines ALL the CPUs this vector
+ * can be associated with. If the vector can be unquely associated with
+ * a single CPU, that CPU will be recorded in hba_eq_hdl[index].cpu.
+ * In addition, the CPU to IO channel mapping will be calculated
+ * and the phba->sli4_hba.cpu_map array will reflect this.
*/
-static int
-lpfc_find_next_cpu(struct lpfc_hba *phba, uint32_t phys_id)
+static void
+lpfc_cpu_affinity_check(struct lpfc_hba *phba, int vectors)
{
struct lpfc_vector_map_info *cpup;
+ int index = 0;
+ int vec = 0;
int cpu;
-
- cpup = phba->sli4_hba.cpu_map;
- for (cpu = 0; cpu < phba->sli4_hba.num_present_cpu; cpu++) {
- /* CPU must be online */
- if (cpu_online(cpu)) {
- if ((cpup->irq == LPFC_VECTOR_MAP_EMPTY) &&
- (lpfc_used_cpu[cpu] == LPFC_VECTOR_MAP_EMPTY) &&
- (cpup->phys_id == phys_id)) {
- return cpu;
- }
- }
- cpup++;
- }
-
- /*
- * If we get here, we have used ALL CPUs for the specific
- * phys_id. Now we need to clear out lpfc_used_cpu and start
- * reusing CPUs.
- */
-
- for (cpu = 0; cpu < phba->sli4_hba.num_present_cpu; cpu++) {
- if (lpfc_used_cpu[cpu] == phys_id)
- lpfc_used_cpu[cpu] = LPFC_VECTOR_MAP_EMPTY;
- }
-
- cpup = phba->sli4_hba.cpu_map;
- for (cpu = 0; cpu < phba->sli4_hba.num_present_cpu; cpu++) {
- /* CPU must be online */
- if (cpu_online(cpu)) {
- if ((cpup->irq == LPFC_VECTOR_MAP_EMPTY) &&
- (cpup->phys_id == phys_id)) {
- return cpu;
- }
- }
- cpup++;
- }
- return LPFC_VECTOR_MAP_EMPTY;
-}
-
-/**
- * lpfc_sli4_set_affinity - Set affinity for HBA IRQ vectors
- * @phba: pointer to lpfc hba data structure.
- * @vectors: number of HBA vectors
- *
- * Affinitize MSIX IRQ vectors to CPUs. Try to equally spread vector
- * affinization across multple physical CPUs (numa nodes).
- * In addition, this routine will assign an IO channel for each CPU
- * to use when issuing I/Os.
- */
-static int
-lpfc_sli4_set_affinity(struct lpfc_hba *phba, int vectors)
-{
- int i, idx, saved_chann, used_chann, cpu, phys_id;
- int max_phys_id, min_phys_id;
- int num_io_channel, first_cpu, chan;
- struct lpfc_vector_map_info *cpup;
#ifdef CONFIG_X86
struct cpuinfo_x86 *cpuinfo;
#endif
- uint8_t chann[LPFC_FCP_IO_CHAN_MAX+1];
-
- /* If there is no mapping, just return */
- if (!phba->cfg_fcp_cpu_map)
- return 1;
/* Init cpu_map array */
memset(phba->sli4_hba.cpu_map, 0xff,
(sizeof(struct lpfc_vector_map_info) *
- phba->sli4_hba.num_present_cpu));
-
- max_phys_id = 0;
- min_phys_id = 0xff;
- phys_id = 0;
- num_io_channel = 0;
- first_cpu = LPFC_VECTOR_MAP_EMPTY;
+ phba->sli4_hba.num_present_cpu));
/* Update CPU map with physical id and core id of each CPU */
cpup = phba->sli4_hba.cpu_map;
cpup->phys_id = 0;
cpup->core_id = 0;
#endif
-
- lpfc_printf_log(phba, KERN_INFO, LOG_INIT,
- "3328 CPU physid %d coreid %d\n",
- cpup->phys_id, cpup->core_id);
-
- if (cpup->phys_id > max_phys_id)
- max_phys_id = cpup->phys_id;
- if (cpup->phys_id < min_phys_id)
- min_phys_id = cpup->phys_id;
+ cpup->channel_id = index; /* For now round robin */
+ cpup->irq = pci_irq_vector(phba->pcidev, vec);
+ vec++;
+ if (vec >= vectors)
+ vec = 0;
+ index++;
+ if (index >= phba->cfg_fcp_io_channel)
+ index = 0;
cpup++;
}
-
- phys_id = min_phys_id;
- /* Now associate the HBA vectors with specific CPUs */
- for (idx = 0; idx < vectors; idx++) {
- cpup = phba->sli4_hba.cpu_map;
- cpu = lpfc_find_next_cpu(phba, phys_id);
- if (cpu == LPFC_VECTOR_MAP_EMPTY) {
-
- /* Try for all phys_id's */
- for (i = 1; i < max_phys_id; i++) {
- phys_id++;
- if (phys_id > max_phys_id)
- phys_id = min_phys_id;
- cpu = lpfc_find_next_cpu(phba, phys_id);
- if (cpu == LPFC_VECTOR_MAP_EMPTY)
- continue;
- goto found;
- }
-
- /* Use round robin for scheduling */
- phba->cfg_fcp_io_sched = LPFC_FCP_SCHED_ROUND_ROBIN;
- chan = 0;
- cpup = phba->sli4_hba.cpu_map;
- for (i = 0; i < phba->sli4_hba.num_present_cpu; i++) {
- cpup->channel_id = chan;
- cpup++;
- chan++;
- if (chan >= phba->cfg_fcp_io_channel)
- chan = 0;
- }
-
- lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
- "3329 Cannot set affinity:"
- "Error mapping vector %d (%d)\n",
- idx, vectors);
- return 0;
- }
-found:
- cpup += cpu;
- if (phba->cfg_fcp_cpu_map == LPFC_DRIVER_CPU_MAP)
- lpfc_used_cpu[cpu] = phys_id;
-
- /* Associate vector with selected CPU */
- cpup->irq = phba->sli4_hba.msix_entries[idx].vector;
-
- /* Associate IO channel with selected CPU */
- cpup->channel_id = idx;
- num_io_channel++;
-
- if (first_cpu == LPFC_VECTOR_MAP_EMPTY)
- first_cpu = cpu;
-
- /* Now affinitize to the selected CPU */
- i = irq_set_affinity_hint(phba->sli4_hba.msix_entries[idx].
- vector, get_cpu_mask(cpu));
-
- lpfc_printf_log(phba, KERN_INFO, LOG_INIT,
- "3330 Set Affinity: CPU %d channel %d "
- "irq %d (%x)\n",
- cpu, cpup->channel_id,
- phba->sli4_hba.msix_entries[idx].vector, i);
-
- /* Spread vector mapping across multple physical CPU nodes */
- phys_id++;
- if (phys_id > max_phys_id)
- phys_id = min_phys_id;
- }
-
- /*
- * Finally fill in the IO channel for any remaining CPUs.
- * At this point, all IO channels have been assigned to a specific
- * MSIx vector, mapped to a specific CPU.
- * Base the remaining IO channel assigned, to IO channels already
- * assigned to other CPUs on the same phys_id.
- */
- for (i = min_phys_id; i <= max_phys_id; i++) {
- /*
- * If there are no io channels already mapped to
- * this phys_id, just round robin thru the io_channels.
- * Setup chann[] for round robin.
- */
- for (idx = 0; idx < phba->cfg_fcp_io_channel; idx++)
- chann[idx] = idx;
-
- saved_chann = 0;
- used_chann = 0;
-
- /*
- * First build a list of IO channels already assigned
- * to this phys_id before reassigning the same IO
- * channels to the remaining CPUs.
- */
- cpup = phba->sli4_hba.cpu_map;
- cpu = first_cpu;
- cpup += cpu;
- for (idx = 0; idx < phba->sli4_hba.num_present_cpu;
- idx++) {
- if (cpup->phys_id == i) {
- /*
- * Save any IO channels that are
- * already mapped to this phys_id.
- */
- if (cpup->irq != LPFC_VECTOR_MAP_EMPTY) {
- if (saved_chann <=
- LPFC_FCP_IO_CHAN_MAX) {
- chann[saved_chann] =
- cpup->channel_id;
- saved_chann++;
- }
- goto out;
- }
-
- /* See if we are using round-robin */
- if (saved_chann == 0)
- saved_chann =
- phba->cfg_fcp_io_channel;
-
- /* Associate next IO channel with CPU */
- cpup->channel_id = chann[used_chann];
- num_io_channel++;
- used_chann++;
- if (used_chann == saved_chann)
- used_chann = 0;
-
- lpfc_printf_log(phba, KERN_INFO, LOG_INIT,
- "3331 Set IO_CHANN "
- "CPU %d channel %d\n",
- idx, cpup->channel_id);
- }
-out:
- cpu++;
- if (cpu >= phba->sli4_hba.num_present_cpu) {
- cpup = phba->sli4_hba.cpu_map;
- cpu = 0;
- } else {
- cpup++;
- }
- }
- }
-
- if (phba->sli4_hba.num_online_cpu != phba->sli4_hba.num_present_cpu) {
- cpup = phba->sli4_hba.cpu_map;
- for (idx = 0; idx < phba->sli4_hba.num_present_cpu; idx++) {
- if (cpup->channel_id == LPFC_VECTOR_MAP_EMPTY) {
- cpup->channel_id = 0;
- num_io_channel++;
-
- lpfc_printf_log(phba, KERN_INFO, LOG_INIT,
- "3332 Assign IO_CHANN "
- "CPU %d channel %d\n",
- idx, cpup->channel_id);
- }
- cpup++;
- }
- }
-
- /* Sanity check */
- if (num_io_channel != phba->sli4_hba.num_present_cpu)
- lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
- "3333 Set affinity mismatch:"
- "%d chann != %d cpus: %d vectors\n",
- num_io_channel, phba->sli4_hba.num_present_cpu,
- vectors);
-
- /* Enable using cpu affinity for scheduling */
- phba->cfg_fcp_io_sched = LPFC_FCP_SCHED_BY_CPU;
- return 1;
}
* @phba: pointer to lpfc hba data structure.
*
* This routine is invoked to enable the MSI-X interrupt vectors to device
- * with SLI-4 interface spec. The kernel function pci_enable_msix_range()
- * is called to enable the MSI-X vectors. The device driver is responsible
- * for calling the individual request_irq() to register each MSI-X vector
- * with a interrupt handler, which is done in this function. Note that
- * later when device is unloading, the driver should always call free_irq()
- * on all MSI-X vectors it has done request_irq() on before calling
- * pci_disable_msix(). Failure to do so results in a BUG_ON() and a device
- * will be left with MSI-X enabled and leaks its vectors.
+ * with SLI-4 interface spec.
*
* Return codes
* 0 - successful
int vectors, rc, index;
/* Set up MSI-X multi-message vectors */
- for (index = 0; index < phba->cfg_fcp_io_channel; index++)
- phba->sli4_hba.msix_entries[index].entry = index;
-
- /* Configure MSI-X capability structure */
- vectors = phba->cfg_fcp_io_channel;
- if (phba->cfg_fof) {
- phba->sli4_hba.msix_entries[index].entry = index;
+ vectors = phba->io_channel_irqs;
+ if (phba->cfg_fof)
vectors++;
- }
- rc = pci_enable_msix_range(phba->pcidev, phba->sli4_hba.msix_entries,
- 2, vectors);
+
+ rc = pci_alloc_irq_vectors(phba->pcidev,
+ (phba->nvmet_support) ? 1 : 2,
+ vectors, PCI_IRQ_MSIX | PCI_IRQ_AFFINITY);
if (rc < 0) {
lpfc_printf_log(phba, KERN_INFO, LOG_INIT,
"0484 PCI enable MSI-X failed (%d)\n", rc);
}
vectors = rc;
- /* Log MSI-X vector assignment */
- for (index = 0; index < vectors; index++)
- lpfc_printf_log(phba, KERN_INFO, LOG_INIT,
- "0489 MSI-X entry[%d]: vector=x%x "
- "message=%d\n", index,
- phba->sli4_hba.msix_entries[index].vector,
- phba->sli4_hba.msix_entries[index].entry);
-
/* Assign MSI-X vectors to interrupt handlers */
for (index = 0; index < vectors; index++) {
memset(&phba->sli4_hba.handler_name[index], 0, 16);
LPFC_SLI4_HANDLER_NAME_SZ,
LPFC_DRIVER_HANDLER_NAME"%d", index);
- phba->sli4_hba.fcp_eq_hdl[index].idx = index;
- phba->sli4_hba.fcp_eq_hdl[index].phba = phba;
- atomic_set(&phba->sli4_hba.fcp_eq_hdl[index].fcp_eq_in_use, 1);
+ phba->sli4_hba.hba_eq_hdl[index].idx = index;
+ phba->sli4_hba.hba_eq_hdl[index].phba = phba;
+ atomic_set(&phba->sli4_hba.hba_eq_hdl[index].hba_eq_in_use, 1);
if (phba->cfg_fof && (index == (vectors - 1)))
- rc = request_irq(
- phba->sli4_hba.msix_entries[index].vector,
+ rc = request_irq(pci_irq_vector(phba->pcidev, index),
&lpfc_sli4_fof_intr_handler, 0,
(char *)&phba->sli4_hba.handler_name[index],
- &phba->sli4_hba.fcp_eq_hdl[index]);
+ &phba->sli4_hba.hba_eq_hdl[index]);
else
- rc = request_irq(
- phba->sli4_hba.msix_entries[index].vector,
+ rc = request_irq(pci_irq_vector(phba->pcidev, index),
&lpfc_sli4_hba_intr_handler, 0,
(char *)&phba->sli4_hba.handler_name[index],
- &phba->sli4_hba.fcp_eq_hdl[index]);
+ &phba->sli4_hba.hba_eq_hdl[index]);
if (rc) {
lpfc_printf_log(phba, KERN_WARNING, LOG_INIT,
"0486 MSI-X fast-path (%d) "
if (phba->cfg_fof)
vectors--;
- if (vectors != phba->cfg_fcp_io_channel) {
+ if (vectors != phba->io_channel_irqs) {
lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
"3238 Reducing IO channels to match number of "
"MSI-X vectors, requested %d got %d\n",
- phba->cfg_fcp_io_channel, vectors);
- phba->cfg_fcp_io_channel = vectors;
+ phba->io_channel_irqs, vectors);
+ if (phba->cfg_fcp_io_channel > vectors)
+ phba->cfg_fcp_io_channel = vectors;
+ if (phba->cfg_nvme_io_channel > vectors)
+ phba->cfg_nvme_io_channel = vectors;
+ if (phba->cfg_fcp_io_channel > phba->cfg_nvme_io_channel)
+ phba->io_channel_irqs = phba->cfg_fcp_io_channel;
+ else
+ phba->io_channel_irqs = phba->cfg_nvme_io_channel;
}
+ lpfc_cpu_affinity_check(phba, vectors);
- if (!shost_use_blk_mq(lpfc_shost_from_vport(phba->pport)))
- lpfc_sli4_set_affinity(phba, vectors);
return rc;
cfg_fail_out:
/* free the irq already requested */
- for (--index; index >= 0; index--) {
- irq_set_affinity_hint(phba->sli4_hba.msix_entries[index].
- vector, NULL);
- free_irq(phba->sli4_hba.msix_entries[index].vector,
- &phba->sli4_hba.fcp_eq_hdl[index]);
- }
+ for (--index; index >= 0; index--)
+ free_irq(pci_irq_vector(phba->pcidev, index),
+ &phba->sli4_hba.hba_eq_hdl[index]);
/* Unconfigure MSI-X capability structure */
- pci_disable_msix(phba->pcidev);
+ pci_free_irq_vectors(phba->pcidev);
vec_fail_out:
return rc;
}
-/**
- * lpfc_sli4_disable_msix - Disable MSI-X interrupt mode to SLI-4 device
- * @phba: pointer to lpfc hba data structure.
- *
- * This routine is invoked to release the MSI-X vectors and then disable the
- * MSI-X interrupt mode to device with SLI-4 interface spec.
- **/
-static void
-lpfc_sli4_disable_msix(struct lpfc_hba *phba)
-{
- int index;
-
- /* Free up MSI-X multi-message vectors */
- for (index = 0; index < phba->cfg_fcp_io_channel; index++) {
- irq_set_affinity_hint(phba->sli4_hba.msix_entries[index].
- vector, NULL);
- free_irq(phba->sli4_hba.msix_entries[index].vector,
- &phba->sli4_hba.fcp_eq_hdl[index]);
- }
- if (phba->cfg_fof) {
- free_irq(phba->sli4_hba.msix_entries[index].vector,
- &phba->sli4_hba.fcp_eq_hdl[index]);
- }
- /* Disable MSI-X */
- pci_disable_msix(phba->pcidev);
-
- return;
-}
-
/**
* lpfc_sli4_enable_msi - Enable MSI interrupt mode to SLI-4 device
* @phba: pointer to lpfc hba data structure.
return rc;
}
- for (index = 0; index < phba->cfg_fcp_io_channel; index++) {
- phba->sli4_hba.fcp_eq_hdl[index].idx = index;
- phba->sli4_hba.fcp_eq_hdl[index].phba = phba;
+ for (index = 0; index < phba->io_channel_irqs; index++) {
+ phba->sli4_hba.hba_eq_hdl[index].idx = index;
+ phba->sli4_hba.hba_eq_hdl[index].phba = phba;
}
if (phba->cfg_fof) {
- phba->sli4_hba.fcp_eq_hdl[index].idx = index;
- phba->sli4_hba.fcp_eq_hdl[index].phba = phba;
+ phba->sli4_hba.hba_eq_hdl[index].idx = index;
+ phba->sli4_hba.hba_eq_hdl[index].phba = phba;
}
return 0;
}
-/**
- * lpfc_sli4_disable_msi - Disable MSI interrupt mode to SLI-4 device
- * @phba: pointer to lpfc hba data structure.
- *
- * This routine is invoked to disable the MSI interrupt mode to device with
- * SLI-4 interface spec. The driver calls free_irq() on MSI vector it has
- * done request_irq() on before calling pci_disable_msi(). Failure to do so
- * results in a BUG_ON() and a device will be left with MSI enabled and leaks
- * its vector.
- **/
-static void
-lpfc_sli4_disable_msi(struct lpfc_hba *phba)
-{
- free_irq(phba->pcidev->irq, phba);
- pci_disable_msi(phba->pcidev);
- return;
-}
-
/**
* lpfc_sli4_enable_intr - Enable device interrupt to SLI-4 device
* @phba: pointer to lpfc hba data structure.
lpfc_sli4_enable_intr(struct lpfc_hba *phba, uint32_t cfg_mode)
{
uint32_t intr_mode = LPFC_INTR_ERROR;
- int retval, index;
+ int retval, idx;
if (cfg_mode == 2) {
/* Preparation before conf_msi mbox cmd */
retval = request_irq(phba->pcidev->irq, lpfc_sli4_intr_handler,
IRQF_SHARED, LPFC_DRIVER_NAME, phba);
if (!retval) {
+ struct lpfc_hba_eq_hdl *eqhdl;
+
/* Indicate initialization to INTx mode */
phba->intr_type = INTx;
intr_mode = 0;
- for (index = 0; index < phba->cfg_fcp_io_channel;
- index++) {
- phba->sli4_hba.fcp_eq_hdl[index].idx = index;
- phba->sli4_hba.fcp_eq_hdl[index].phba = phba;
- atomic_set(&phba->sli4_hba.fcp_eq_hdl[index].
- fcp_eq_in_use, 1);
+
+ for (idx = 0; idx < phba->io_channel_irqs; idx++) {
+ eqhdl = &phba->sli4_hba.hba_eq_hdl[idx];
+ eqhdl->idx = idx;
+ eqhdl->phba = phba;
+ atomic_set(&eqhdl->hba_eq_in_use, 1);
}
if (phba->cfg_fof) {
- phba->sli4_hba.fcp_eq_hdl[index].idx = index;
- phba->sli4_hba.fcp_eq_hdl[index].phba = phba;
- atomic_set(&phba->sli4_hba.fcp_eq_hdl[index].
- fcp_eq_in_use, 1);
+ eqhdl = &phba->sli4_hba.hba_eq_hdl[idx];
+ eqhdl->idx = idx;
+ eqhdl->phba = phba;
+ atomic_set(&eqhdl->hba_eq_in_use, 1);
}
}
}
lpfc_sli4_disable_intr(struct lpfc_hba *phba)
{
/* Disable the currently initialized interrupt mode */
- if (phba->intr_type == MSIX)
- lpfc_sli4_disable_msix(phba);
- else if (phba->intr_type == MSI)
- lpfc_sli4_disable_msi(phba);
- else if (phba->intr_type == INTx)
+ if (phba->intr_type == MSIX) {
+ int index;
+
+ /* Free up MSI-X multi-message vectors */
+ for (index = 0; index < phba->io_channel_irqs; index++)
+ free_irq(pci_irq_vector(phba->pcidev, index),
+ &phba->sli4_hba.hba_eq_hdl[index]);
+
+ if (phba->cfg_fof)
+ free_irq(pci_irq_vector(phba->pcidev, index),
+ &phba->sli4_hba.hba_eq_hdl[index]);
+ } else {
free_irq(phba->pcidev->irq, phba);
+ }
+
+ pci_free_irq_vectors(phba->pcidev);
/* Reset interrupt management states */
phba->intr_type = NONE;
phba->sli.slistat.sli_intr = 0;
-
- return;
}
/**
lpfc_sli4_xri_exchange_busy_wait(struct lpfc_hba *phba)
{
int wait_time = 0;
- int fcp_xri_cmpl = list_empty(&phba->sli4_hba.lpfc_abts_scsi_buf_list);
+ int nvme_xri_cmpl = 1;
+ int fcp_xri_cmpl = 1;
int els_xri_cmpl = list_empty(&phba->sli4_hba.lpfc_abts_els_sgl_list);
+ int nvmet_xri_cmpl =
+ list_empty(&phba->sli4_hba.lpfc_abts_nvmet_sgl_list);
+
+ if (phba->cfg_enable_fc4_type & LPFC_ENABLE_FCP)
+ fcp_xri_cmpl =
+ list_empty(&phba->sli4_hba.lpfc_abts_scsi_buf_list);
+ if (phba->cfg_enable_fc4_type & LPFC_ENABLE_NVME)
+ nvme_xri_cmpl =
+ list_empty(&phba->sli4_hba.lpfc_abts_nvme_buf_list);
- while (!fcp_xri_cmpl || !els_xri_cmpl) {
+ while (!fcp_xri_cmpl || !els_xri_cmpl || !nvme_xri_cmpl ||
+ !nvmet_xri_cmpl) {
if (wait_time > LPFC_XRI_EXCH_BUSY_WAIT_TMO) {
+ if (!nvme_xri_cmpl)
+ lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+ "6100 NVME XRI exchange busy "
+ "wait time: %d seconds.\n",
+ wait_time/1000);
if (!fcp_xri_cmpl)
lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
"2877 FCP XRI exchange busy "
msleep(LPFC_XRI_EXCH_BUSY_WAIT_T1);
wait_time += LPFC_XRI_EXCH_BUSY_WAIT_T1;
}
- fcp_xri_cmpl =
- list_empty(&phba->sli4_hba.lpfc_abts_scsi_buf_list);
+ if (phba->cfg_enable_fc4_type & LPFC_ENABLE_NVME)
+ nvme_xri_cmpl = list_empty(
+ &phba->sli4_hba.lpfc_abts_nvme_buf_list);
+
+ if (phba->cfg_enable_fc4_type & LPFC_ENABLE_FCP)
+ fcp_xri_cmpl = list_empty(
+ &phba->sli4_hba.lpfc_abts_scsi_buf_list);
+
els_xri_cmpl =
list_empty(&phba->sli4_hba.lpfc_abts_els_sgl_list);
+
+ nvmet_xri_cmpl =
+ list_empty(&phba->sli4_hba.lpfc_abts_nvmet_sgl_list);
}
}
sli4_params->wqsize = bf_get(cfg_wqsize, mbx_sli4_parameters);
sli4_params->sgl_pages_max = bf_get(cfg_sgl_page_cnt,
mbx_sli4_parameters);
+ sli4_params->wqpcnt = bf_get(cfg_wqpcnt, mbx_sli4_parameters);
sli4_params->sgl_pp_align = bf_get(cfg_sgl_pp_align,
mbx_sli4_parameters);
phba->sli4_hba.extents_in_use = bf_get(cfg_ext, mbx_sli4_parameters);
phba->sli4_hba.rpi_hdrs_in_use = bf_get(cfg_hdrr, mbx_sli4_parameters);
+ phba->nvme_support = (bf_get(cfg_nvme, mbx_sli4_parameters) &&
+ bf_get(cfg_xib, mbx_sli4_parameters));
+
+ if ((phba->cfg_enable_fc4_type == LPFC_ENABLE_FCP) ||
+ !phba->nvme_support) {
+ phba->nvme_support = 0;
+ phba->nvmet_support = 0;
+ phba->cfg_nvmet_mrq = 0;
+ phba->cfg_nvme_io_channel = 0;
+ phba->io_channel_irqs = phba->cfg_fcp_io_channel;
+ lpfc_printf_log(phba, KERN_ERR, LOG_INIT | LOG_NVME,
+ "6101 Disabling NVME support: "
+ "Not supported by firmware: %d %d\n",
+ bf_get(cfg_nvme, mbx_sli4_parameters),
+ bf_get(cfg_xib, mbx_sli4_parameters));
+
+ /* If firmware doesn't support NVME, just use SCSI support */
+ if (!(phba->cfg_enable_fc4_type & LPFC_ENABLE_FCP))
+ return -ENODEV;
+ phba->cfg_enable_fc4_type = LPFC_ENABLE_FCP;
+ }
+
+ if (bf_get(cfg_xib, mbx_sli4_parameters) && phba->cfg_suppress_rsp)
+ phba->sli.sli_flag |= LPFC_SLI_SUPPRESS_RSP;
/* Make sure that sge_supp_len can be handled by the driver */
if (sli4_params->sge_supp_len > LPFC_MAX_SGE_SIZE)
goto out_disable_pci_dev;
}
- /* Set up phase-1 common device driver resources */
- error = lpfc_setup_driver_resource_phase1(phba);
- if (error) {
- lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
- "1403 Failed to set up driver resource.\n");
- goto out_unset_pci_mem_s3;
- }
-
/* Set up SLI-3 specific device driver resources */
error = lpfc_sli_driver_resource_setup(phba);
if (error) {
/* Remove FC host and then SCSI host with the physical port */
fc_remove_host(shost);
scsi_remove_host(shost);
+
+ /* Perform ndlp cleanup on the physical port. The nvme and nvmet
+ * localports are destroyed after to cleanup all transport memory.
+ */
lpfc_cleanup(vport);
+ lpfc_nvmet_destroy_targetport(phba);
+ lpfc_nvme_destroy_localport(vport);
/*
* Bring down the SLI Layer. This step disable all interrupts,
return 0;
}
+/**
+ * lpfc_sli4_get_iocb_cnt - Calculate the # of total IOCBs to reserve
+ * @phba: pointer to lpfc hba data structure.
+ *
+ * returns the number of ELS/CT + NVMET IOCBs to reserve
+ **/
+int
+lpfc_sli4_get_iocb_cnt(struct lpfc_hba *phba)
+{
+ int max_xri = lpfc_sli4_get_els_iocb_cnt(phba);
+
+ if (phba->nvmet_support)
+ max_xri += LPFC_NVMET_BUF_POST;
+ return max_xri;
+}
+
+
/**
* lpfc_write_firmware - attempt to write a firmware image to the port
* @fw: pointer to firmware image returned from request_firmware.
struct Scsi_Host *shost = NULL;
int error;
uint32_t cfg_mode, intr_mode;
- int adjusted_fcp_io_channel;
/* Allocate memory for HBA structure */
phba = lpfc_hba_alloc(pdev);
goto out_disable_pci_dev;
}
- /* Set up phase-1 common device driver resources */
- error = lpfc_setup_driver_resource_phase1(phba);
- if (error) {
- lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
- "1411 Failed to set up driver resource.\n");
- goto out_unset_pci_mem_s4;
- }
-
/* Set up SLI-4 Specific device driver resources */
error = lpfc_sli4_driver_resource_setup(phba);
if (error) {
/* Put device to a known state before enabling interrupt */
lpfc_stop_port(phba);
+
/* Configure and enable interrupt */
intr_mode = lpfc_sli4_enable_intr(phba, cfg_mode);
if (intr_mode == LPFC_INTR_ERROR) {
goto out_free_sysfs_attr;
}
/* Default to single EQ for non-MSI-X */
- if (phba->intr_type != MSIX)
- adjusted_fcp_io_channel = 1;
- else
- adjusted_fcp_io_channel = phba->cfg_fcp_io_channel;
- phba->cfg_fcp_io_channel = adjusted_fcp_io_channel;
+ if (phba->intr_type != MSIX) {
+ if (phba->cfg_enable_fc4_type & LPFC_ENABLE_FCP)
+ phba->cfg_fcp_io_channel = 1;
+ if (phba->cfg_enable_fc4_type & LPFC_ENABLE_NVME) {
+ phba->cfg_nvme_io_channel = 1;
+ if (phba->nvmet_support)
+ phba->cfg_nvmet_mrq = 1;
+ }
+ phba->io_channel_irqs = 1;
+ }
+
/* Set up SLI-4 HBA */
if (lpfc_sli4_hba_setup(phba)) {
lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
/* Perform post initialization setup */
lpfc_post_init_setup(phba);
+ /* NVME support in FW earlier in the driver load corrects the
+ * FC4 type making a check for nvme_support unnecessary.
+ */
+ if ((phba->nvmet_support == 0) &&
+ (phba->cfg_enable_fc4_type & LPFC_ENABLE_NVME)) {
+ /* Create NVME binding with nvme_fc_transport. This
+ * ensures the vport is initialized.
+ */
+ error = lpfc_nvme_create_localport(vport);
+ if (error) {
+ lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+ "6004 NVME registration failed, "
+ "error x%x\n",
+ error);
+ goto out_disable_intr;
+ }
+ }
+
/* check for firmware upgrade or downgrade */
if (phba->cfg_request_firmware_upgrade)
lpfc_sli4_request_firmware_update(phba, INT_FW_UPGRADE);
fc_remove_host(shost);
scsi_remove_host(shost);
- /* Perform cleanup on the physical port */
+ /* Perform ndlp cleanup on the physical port. The nvme and nvmet
+ * localports are destroyed after to cleanup all transport memory.
+ */
lpfc_cleanup(vport);
+ lpfc_nvmet_destroy_targetport(phba);
+ lpfc_nvme_destroy_localport(vport);
/*
* Bring down the SLI Layer. This step disables all interrupts,
* buffers are released to their corresponding pools here.
*/
lpfc_scsi_free(phba);
+ lpfc_nvme_free(phba);
+ lpfc_free_iocb_list(phba);
lpfc_sli4_driver_resource_unset(phba);
int
lpfc_fof_queue_setup(struct lpfc_hba *phba)
{
- struct lpfc_sli *psli = &phba->sli;
+ struct lpfc_sli_ring *pring;
int rc;
rc = lpfc_eq_create(phba, phba->sli4_hba.fof_eq, LPFC_MAX_IMAX);
if (rc)
goto out_oas_wq;
- phba->sli4_hba.oas_cq->pring = &psli->ring[LPFC_FCP_OAS_RING];
- phba->sli4_hba.oas_ring = &psli->ring[LPFC_FCP_OAS_RING];
+ /* Bind this CQ/WQ to the NVME ring */
+ pring = phba->sli4_hba.oas_wq->pring;
+ pring->sli.sli4.wqp =
+ (void *)phba->sli4_hba.oas_wq;
+ phba->sli4_hba.oas_cq->pring = pring;
}
return 0;
goto out_error;
phba->sli4_hba.oas_wq = qdesc;
+ list_add_tail(&qdesc->wq_list, &phba->sli4_hba.lpfc_wq_list);
}
return 0;
static int __init
lpfc_init(void)
{
- int cpu;
int error = 0;
printk(LPFC_MODULE_DESC "\n");
/* Initialize in case vector mapping is needed */
lpfc_used_cpu = NULL;
- lpfc_present_cpu = 0;
- for_each_present_cpu(cpu)
- lpfc_present_cpu++;
+ lpfc_present_cpu = num_present_cpus();
error = pci_register_driver(&lpfc_driver);
if (error) {
module_exit(lpfc_exit);
MODULE_LICENSE("GPL");
MODULE_DESCRIPTION(LPFC_MODULE_DESC);
-MODULE_AUTHOR("Emulex Corporation - tech.support@emulex.com");
+MODULE_AUTHOR("Broadcom");
MODULE_VERSION("0:" LPFC_DRIVER_VERSION);
/*******************************************************************
* This file is part of the Emulex Linux Device Driver for *
* Fibre Channel Host Bus Adapters. *
+ * Copyright (C) 2017 Broadcom. All Rights Reserved. The term *
+ * “Broadcom” refers to Broadcom Limited and/or its subsidiaries. *
* Copyright (C) 2004-2009 Emulex. All rights reserved. *
* EMULEX and SLI are trademarks of Emulex. *
- * www.emulex.com *
+ * www.broadcom.com *
* *
* This program is free software; you can redistribute it and/or *
* modify it under the terms of version 2 of the GNU General *
#define LOG_FIP 0x00020000 /* FIP events */
#define LOG_FCP_UNDER 0x00040000 /* FCP underruns errors */
#define LOG_SCSI_CMD 0x00080000 /* ALL SCSI commands */
+#define LOG_NVME 0x00100000 /* NVME general events. */
+#define LOG_NVME_DISC 0x00200000 /* NVME Discovery/Connect events. */
+#define LOG_NVME_ABTS 0x00400000 /* NVME ABTS events. */
+#define LOG_NVME_IOERR 0x00800000 /* NVME IO Error events. */
#define LOG_ALL_MSG 0xffffffff /* LOG all messages */
#define lpfc_printf_vlog(vport, level, mask, fmt, arg...) \
/*******************************************************************
* This file is part of the Emulex Linux Device Driver for *
* Fibre Channel Host Bus Adapters. *
+ * Copyright (C) 2017 Broadcom. All Rights Reserved. The term *
+ * “Broadcom” refers to Broadcom Limited and/or its subsidiaries. *
* Copyright (C) 2004-2016 Emulex. All rights reserved. *
* EMULEX and SLI are trademarks of Emulex. *
- * www.emulex.com *
+ * www.broadcom.com *
* Portions Copyright (C) 2004-2005 Christoph Hellwig *
* *
* This program is free software; you can redistribute it and/or *
pcbp->maxRing = (psli->num_rings - 1);
for (i = 0; i < psli->num_rings; i++) {
- pring = &psli->ring[i];
+ pring = &psli->sli3_ring[i];
pring->sli.sli3.sizeCiocb =
phba->sli_rev == 3 ? SLI3_IOCB_CMD_SIZE :
mb->un.varCfgRing.recvNotify = 1;
psli = &phba->sli;
- pring = &psli->ring[ring];
+ pring = &psli->sli3_ring[ring];
mb->un.varCfgRing.numMask = pring->num_mask;
mb->mbxCommand = MBX_CONFIG_RING;
mb->mbxOwner = OWN_HOST;
if (phba->max_vpi && phba->cfg_enable_npiv)
bf_set(lpfc_mbx_rq_ftr_rq_npiv, &mboxq->u.mqe.un.req_ftrs, 1);
+ if (phba->nvmet_support)
+ bf_set(lpfc_mbx_rq_ftr_rq_mrqp, &mboxq->u.mqe.un.req_ftrs, 1);
+
return;
}
memset(mbox, 0, sizeof(*mbox));
reg_fcfi = &mbox->u.mqe.un.reg_fcfi;
bf_set(lpfc_mqe_command, &mbox->u.mqe, MBX_REG_FCFI);
- bf_set(lpfc_reg_fcfi_rq_id0, reg_fcfi, phba->sli4_hba.hdr_rq->queue_id);
- bf_set(lpfc_reg_fcfi_rq_id1, reg_fcfi, REG_FCF_INVALID_QID);
+ if (phba->nvmet_support == 0) {
+ bf_set(lpfc_reg_fcfi_rq_id0, reg_fcfi,
+ phba->sli4_hba.hdr_rq->queue_id);
+ /* Match everything - rq_id0 */
+ bf_set(lpfc_reg_fcfi_type_match0, reg_fcfi, 0);
+ bf_set(lpfc_reg_fcfi_type_mask0, reg_fcfi, 0);
+ bf_set(lpfc_reg_fcfi_rctl_match0, reg_fcfi, 0);
+ bf_set(lpfc_reg_fcfi_rctl_mask0, reg_fcfi, 0);
+
+ bf_set(lpfc_reg_fcfi_rq_id1, reg_fcfi, REG_FCF_INVALID_QID);
+
+ /* addr mode is bit wise inverted value of fcf addr_mode */
+ bf_set(lpfc_reg_fcfi_mam, reg_fcfi,
+ (~phba->fcf.addr_mode) & 0x3);
+ } else {
+ /* This is ONLY for NVMET MRQ == 1 */
+ if (phba->cfg_nvmet_mrq != 1)
+ return;
+
+ bf_set(lpfc_reg_fcfi_rq_id0, reg_fcfi,
+ phba->sli4_hba.nvmet_mrq_hdr[0]->queue_id);
+ /* Match type FCP - rq_id0 */
+ bf_set(lpfc_reg_fcfi_type_match0, reg_fcfi, FC_TYPE_FCP);
+ bf_set(lpfc_reg_fcfi_type_mask0, reg_fcfi, 0xff);
+ bf_set(lpfc_reg_fcfi_rctl_match0, reg_fcfi,
+ FC_RCTL_DD_UNSOL_CMD);
+
+ bf_set(lpfc_reg_fcfi_rq_id1, reg_fcfi,
+ phba->sli4_hba.hdr_rq->queue_id);
+ /* Match everything else - rq_id1 */
+ bf_set(lpfc_reg_fcfi_type_match1, reg_fcfi, 0);
+ bf_set(lpfc_reg_fcfi_type_mask1, reg_fcfi, 0);
+ bf_set(lpfc_reg_fcfi_rctl_match1, reg_fcfi, 0);
+ bf_set(lpfc_reg_fcfi_rctl_mask1, reg_fcfi, 0);
+ }
bf_set(lpfc_reg_fcfi_rq_id2, reg_fcfi, REG_FCF_INVALID_QID);
bf_set(lpfc_reg_fcfi_rq_id3, reg_fcfi, REG_FCF_INVALID_QID);
bf_set(lpfc_reg_fcfi_info_index, reg_fcfi,
phba->fcf.current_rec.fcf_indx);
- /* reg_fcf addr mode is bit wise inverted value of fcf addr_mode */
- bf_set(lpfc_reg_fcfi_mam, reg_fcfi, (~phba->fcf.addr_mode) & 0x3);
if (phba->fcf.current_rec.vlan_id != LPFC_FCOE_NULL_VID) {
bf_set(lpfc_reg_fcfi_vv, reg_fcfi, 1);
bf_set(lpfc_reg_fcfi_vlan_tag, reg_fcfi,
}
}
+/**
+ * lpfc_reg_fcfi_mrq - Initialize the REG_FCFI_MRQ mailbox command
+ * @phba: pointer to the hba structure containing the FCF index and RQ ID.
+ * @mbox: pointer to lpfc mbox command to initialize.
+ * @mode: 0 to register FCFI, 1 to register MRQs
+ *
+ * The REG_FCFI_MRQ mailbox command supports Fibre Channel Forwarders (FCFs).
+ * The SLI Host uses the command to activate an FCF after it has acquired FCF
+ * information via a READ_FCF mailbox command. This mailbox command also is used
+ * to indicate where received unsolicited frames from this FCF will be sent. By
+ * default this routine will set up the FCF to forward all unsolicited frames
+ * the the RQ ID passed in the @phba. This can be overridden by the caller for
+ * more complicated setups.
+ **/
+void
+lpfc_reg_fcfi_mrq(struct lpfc_hba *phba, struct lpfcMboxq *mbox, int mode)
+{
+ struct lpfc_mbx_reg_fcfi_mrq *reg_fcfi;
+
+ /* This is ONLY for MRQ */
+ if (phba->cfg_nvmet_mrq <= 1)
+ return;
+
+ memset(mbox, 0, sizeof(*mbox));
+ reg_fcfi = &mbox->u.mqe.un.reg_fcfi_mrq;
+ bf_set(lpfc_mqe_command, &mbox->u.mqe, MBX_REG_FCFI_MRQ);
+ if (mode == 0) {
+ bf_set(lpfc_reg_fcfi_mrq_info_index, reg_fcfi,
+ phba->fcf.current_rec.fcf_indx);
+ if (phba->fcf.current_rec.vlan_id != LPFC_FCOE_NULL_VID) {
+ bf_set(lpfc_reg_fcfi_mrq_vv, reg_fcfi, 1);
+ bf_set(lpfc_reg_fcfi_mrq_vlan_tag, reg_fcfi,
+ phba->fcf.current_rec.vlan_id);
+ }
+ return;
+ }
+
+ bf_set(lpfc_reg_fcfi_mrq_rq_id0, reg_fcfi,
+ phba->sli4_hba.nvmet_mrq_hdr[0]->queue_id);
+ /* Match NVME frames of type FCP (protocol NVME) - rq_id0 */
+ bf_set(lpfc_reg_fcfi_mrq_type_match0, reg_fcfi, FC_TYPE_FCP);
+ bf_set(lpfc_reg_fcfi_mrq_type_mask0, reg_fcfi, 0xff);
+ bf_set(lpfc_reg_fcfi_mrq_rctl_match0, reg_fcfi, FC_RCTL_DD_UNSOL_CMD);
+ bf_set(lpfc_reg_fcfi_mrq_rctl_mask0, reg_fcfi, 0xff);
+ bf_set(lpfc_reg_fcfi_mrq_ptc0, reg_fcfi, 1);
+ bf_set(lpfc_reg_fcfi_mrq_pt0, reg_fcfi, 1);
+
+ bf_set(lpfc_reg_fcfi_mrq_policy, reg_fcfi, 3); /* NVME connection id */
+ bf_set(lpfc_reg_fcfi_mrq_mode, reg_fcfi, 1);
+ bf_set(lpfc_reg_fcfi_mrq_filter, reg_fcfi, 1); /* rq_id0 */
+ bf_set(lpfc_reg_fcfi_mrq_npairs, reg_fcfi, phba->cfg_nvmet_mrq);
+
+ bf_set(lpfc_reg_fcfi_mrq_rq_id1, reg_fcfi,
+ phba->sli4_hba.hdr_rq->queue_id);
+ /* Match everything - rq_id1 */
+ bf_set(lpfc_reg_fcfi_mrq_type_match1, reg_fcfi, 0);
+ bf_set(lpfc_reg_fcfi_mrq_type_mask1, reg_fcfi, 0);
+ bf_set(lpfc_reg_fcfi_mrq_rctl_match1, reg_fcfi, 0);
+ bf_set(lpfc_reg_fcfi_mrq_rctl_mask1, reg_fcfi, 0);
+
+ bf_set(lpfc_reg_fcfi_mrq_rq_id2, reg_fcfi, REG_FCF_INVALID_QID);
+ bf_set(lpfc_reg_fcfi_mrq_rq_id3, reg_fcfi, REG_FCF_INVALID_QID);
+}
+
/**
* lpfc_unreg_fcfi - Initialize the UNREG_FCFI mailbox command
* @mbox: pointer to lpfc mbox command to initialize.
/*******************************************************************
* This file is part of the Emulex Linux Device Driver for *
* Fibre Channel Host Bus Adapters. *
+ * Copyright (C) 2017 Broadcom. All Rights Reserved. The term *
+ * “Broadcom” refers to Broadcom Limited and/or its subsidiaries. *
* Copyright (C) 2004-2014 Emulex. All rights reserved. *
* EMULEX and SLI are trademarks of Emulex. *
- * www.emulex.com *
+ * www.broadcom.com *
* Portions Copyright (C) 2004-2005 Christoph Hellwig *
* *
* This program is free software; you can redistribute it and/or *
#include <linux/pci.h>
#include <linux/interrupt.h>
+#include <scsi/scsi.h>
#include <scsi/scsi_device.h>
#include <scsi/scsi_transport_fc.h>
+#include <scsi/fc/fc_fs.h>
-#include <scsi/scsi.h>
+#include <linux/nvme-fc-driver.h>
#include "lpfc_hw4.h"
#include "lpfc_hw.h"
#include "lpfc_sli4.h"
#include "lpfc_nl.h"
#include "lpfc_disc.h"
-#include "lpfc_scsi.h"
#include "lpfc.h"
+#include "lpfc_scsi.h"
+#include "lpfc_nvme.h"
+#include "lpfc_nvmet.h"
#include "lpfc_crtn.h"
#include "lpfc_logmsg.h"
* lpfc_mem_alloc - create and allocate all PCI and memory pools
* @phba: HBA to allocate pools for
*
- * Description: Creates and allocates PCI pools lpfc_scsi_dma_buf_pool,
+ * Description: Creates and allocates PCI pools lpfc_sg_dma_buf_pool,
* lpfc_mbuf_pool, lpfc_hrb_pool. Creates and allocates kmalloc-backed mempools
* for LPFC_MBOXQ_t and lpfc_nodelist. Also allocates the VPI bitmask.
*
else
i = SLI4_PAGE_SIZE;
- phba->lpfc_scsi_dma_buf_pool =
- pci_pool_create("lpfc_scsi_dma_buf_pool",
- phba->pcidev,
- phba->cfg_sg_dma_buf_size,
- i,
- 0);
+ phba->lpfc_sg_dma_buf_pool =
+ pci_pool_create("lpfc_sg_dma_buf_pool",
+ phba->pcidev,
+ phba->cfg_sg_dma_buf_size,
+ i, 0);
+ if (!phba->lpfc_sg_dma_buf_pool)
+ goto fail;
+
} else {
- phba->lpfc_scsi_dma_buf_pool =
- pci_pool_create("lpfc_scsi_dma_buf_pool",
- phba->pcidev, phba->cfg_sg_dma_buf_size,
- align, 0);
- }
+ phba->lpfc_sg_dma_buf_pool =
+ pci_pool_create("lpfc_sg_dma_buf_pool",
+ phba->pcidev, phba->cfg_sg_dma_buf_size,
+ align, 0);
- if (!phba->lpfc_scsi_dma_buf_pool)
- goto fail;
+ if (!phba->lpfc_sg_dma_buf_pool)
+ goto fail;
+ }
phba->lpfc_mbuf_pool = pci_pool_create("lpfc_mbuf_pool", phba->pcidev,
LPFC_BPL_SIZE,
LPFC_DEVICE_DATA_POOL_SIZE,
sizeof(struct lpfc_device_data));
if (!phba->device_data_mem_pool)
- goto fail_free_hrb_pool;
+ goto fail_free_drb_pool;
} else {
phba->device_data_mem_pool = NULL;
}
return 0;
+fail_free_drb_pool:
+ pci_pool_destroy(phba->lpfc_drb_pool);
+ phba->lpfc_drb_pool = NULL;
fail_free_hrb_pool:
pci_pool_destroy(phba->lpfc_hrb_pool);
phba->lpfc_hrb_pool = NULL;
pci_pool_destroy(phba->lpfc_mbuf_pool);
phba->lpfc_mbuf_pool = NULL;
fail_free_dma_buf_pool:
- pci_pool_destroy(phba->lpfc_scsi_dma_buf_pool);
- phba->lpfc_scsi_dma_buf_pool = NULL;
+ pci_pool_destroy(phba->lpfc_sg_dma_buf_pool);
+ phba->lpfc_sg_dma_buf_pool = NULL;
fail:
return -ENOMEM;
}
if (phba->lpfc_hrb_pool)
pci_pool_destroy(phba->lpfc_hrb_pool);
phba->lpfc_hrb_pool = NULL;
+ if (phba->txrdy_payload_pool)
+ pci_pool_destroy(phba->txrdy_payload_pool);
+ phba->txrdy_payload_pool = NULL;
if (phba->lpfc_hbq_pool)
pci_pool_destroy(phba->lpfc_hbq_pool);
phba->lpfc_mbuf_pool = NULL;
/* Free DMA buffer memory pool */
- pci_pool_destroy(phba->lpfc_scsi_dma_buf_pool);
- phba->lpfc_scsi_dma_buf_pool = NULL;
+ pci_pool_destroy(phba->lpfc_sg_dma_buf_pool);
+ phba->lpfc_sg_dma_buf_pool = NULL;
/* Free Device Data memory pool */
if (phba->device_data_mem_pool) {
* @phba: HBA to free memory for
*
* Description: Free memory from PCI and driver memory pools and also those
- * used : lpfc_scsi_dma_buf_pool, lpfc_mbuf_pool, lpfc_hrb_pool. Frees
+ * used : lpfc_sg_dma_buf_pool, lpfc_mbuf_pool, lpfc_hrb_pool. Frees
* kmalloc-backed mempools for LPFC_MBOXQ_t and lpfc_nodelist. Also frees
* the VPI bitmask.
*
return;
}
+/**
+ * lpfc_nvmet_buf_alloc - Allocate an nvmet_buf from the
+ * lpfc_sg_dma_buf_pool PCI pool
+ * @phba: HBA which owns the pool to allocate from
+ * @mem_flags: indicates if this is a priority (MEM_PRI) allocation
+ * @handle: used to return the DMA-mapped address of the nvmet_buf
+ *
+ * Description: Allocates a DMA-mapped buffer from the lpfc_sg_dma_buf_pool
+ * PCI pool. Allocates from generic pci_pool_alloc function.
+ *
+ * Returns:
+ * pointer to the allocated nvmet_buf on success
+ * NULL on failure
+ **/
+void *
+lpfc_nvmet_buf_alloc(struct lpfc_hba *phba, int mem_flags, dma_addr_t *handle)
+{
+ void *ret;
+
+ ret = pci_pool_alloc(phba->lpfc_sg_dma_buf_pool, GFP_KERNEL, handle);
+ return ret;
+}
+
+/**
+ * lpfc_nvmet_buf_free - Free an nvmet_buf from the lpfc_sg_dma_buf_pool
+ * PCI pool
+ * @phba: HBA which owns the pool to return to
+ * @virt: nvmet_buf to free
+ * @dma: the DMA-mapped address of the lpfc_sg_dma_buf_pool to be freed
+ *
+ * Returns: None
+ **/
+void
+lpfc_nvmet_buf_free(struct lpfc_hba *phba, void *virt, dma_addr_t dma)
+{
+ pci_pool_free(phba->lpfc_sg_dma_buf_pool, virt, dma);
+}
+
/**
* lpfc_els_hbq_alloc - Allocate an HBQ buffer
* @phba: HBA to allocate HBQ buffer for
kfree(hbqbp);
return NULL;
}
- hbqbp->size = LPFC_BPL_SIZE;
+ hbqbp->total_size = LPFC_BPL_SIZE;
return hbqbp;
}
kfree(dma_buf);
return NULL;
}
- dma_buf->size = LPFC_BPL_SIZE;
+ dma_buf->total_size = LPFC_DATA_BUF_SIZE;
return dma_buf;
}
pci_pool_free(phba->lpfc_hrb_pool, dmab->hbuf.virt, dmab->hbuf.phys);
pci_pool_free(phba->lpfc_drb_pool, dmab->dbuf.virt, dmab->dbuf.phys);
kfree(dmab);
- return;
+}
+
+/**
+ * lpfc_sli4_nvmet_alloc - Allocate an SLI4 Receive buffer
+ * @phba: HBA to allocate a receive buffer for
+ *
+ * Description: Allocates a DMA-mapped receive buffer from the lpfc_hrb_pool PCI
+ * pool along a non-DMA-mapped container for it.
+ *
+ * Notes: Not interrupt-safe. Must be called with no locks held.
+ *
+ * Returns:
+ * pointer to HBQ on success
+ * NULL on failure
+ **/
+struct rqb_dmabuf *
+lpfc_sli4_nvmet_alloc(struct lpfc_hba *phba)
+{
+ struct rqb_dmabuf *dma_buf;
+ struct lpfc_iocbq *nvmewqe;
+ union lpfc_wqe128 *wqe;
+
+ dma_buf = kzalloc(sizeof(struct rqb_dmabuf), GFP_KERNEL);
+ if (!dma_buf)
+ return NULL;
+
+ dma_buf->hbuf.virt = pci_pool_alloc(phba->lpfc_hrb_pool, GFP_KERNEL,
+ &dma_buf->hbuf.phys);
+ if (!dma_buf->hbuf.virt) {
+ kfree(dma_buf);
+ return NULL;
+ }
+ dma_buf->dbuf.virt = pci_pool_alloc(phba->lpfc_drb_pool, GFP_KERNEL,
+ &dma_buf->dbuf.phys);
+ if (!dma_buf->dbuf.virt) {
+ pci_pool_free(phba->lpfc_hrb_pool, dma_buf->hbuf.virt,
+ dma_buf->hbuf.phys);
+ kfree(dma_buf);
+ return NULL;
+ }
+ dma_buf->total_size = LPFC_DATA_BUF_SIZE;
+
+ dma_buf->context = kzalloc(sizeof(struct lpfc_nvmet_rcv_ctx),
+ GFP_KERNEL);
+ if (!dma_buf->context) {
+ pci_pool_free(phba->lpfc_drb_pool, dma_buf->dbuf.virt,
+ dma_buf->dbuf.phys);
+ pci_pool_free(phba->lpfc_hrb_pool, dma_buf->hbuf.virt,
+ dma_buf->hbuf.phys);
+ kfree(dma_buf);
+ return NULL;
+ }
+
+ dma_buf->iocbq = lpfc_sli_get_iocbq(phba);
+ dma_buf->iocbq->iocb_flag = LPFC_IO_NVMET;
+ if (!dma_buf->iocbq) {
+ kfree(dma_buf->context);
+ pci_pool_free(phba->lpfc_drb_pool, dma_buf->dbuf.virt,
+ dma_buf->dbuf.phys);
+ pci_pool_free(phba->lpfc_hrb_pool, dma_buf->hbuf.virt,
+ dma_buf->hbuf.phys);
+ kfree(dma_buf);
+ lpfc_printf_log(phba, KERN_ERR, LOG_NVME,
+ "2621 Ran out of nvmet iocb/WQEs\n");
+ return NULL;
+ }
+ nvmewqe = dma_buf->iocbq;
+ wqe = (union lpfc_wqe128 *)&nvmewqe->wqe;
+ /* Initialize WQE */
+ memset(wqe, 0, sizeof(union lpfc_wqe));
+ /* Word 7 */
+ bf_set(wqe_ct, &wqe->generic.wqe_com, SLI4_CT_RPI);
+ bf_set(wqe_class, &wqe->generic.wqe_com, CLASS3);
+ bf_set(wqe_pu, &wqe->generic.wqe_com, 1);
+ /* Word 10 */
+ bf_set(wqe_nvme, &wqe->fcp_tsend.wqe_com, 1);
+ bf_set(wqe_ebde_cnt, &wqe->generic.wqe_com, 0);
+ bf_set(wqe_qosd, &wqe->generic.wqe_com, 0);
+
+ dma_buf->iocbq->context1 = NULL;
+ spin_lock(&phba->sli4_hba.sgl_list_lock);
+ dma_buf->sglq = __lpfc_sli_get_nvmet_sglq(phba, dma_buf->iocbq);
+ spin_unlock(&phba->sli4_hba.sgl_list_lock);
+ if (!dma_buf->sglq) {
+ lpfc_sli_release_iocbq(phba, dma_buf->iocbq);
+ kfree(dma_buf->context);
+ pci_pool_free(phba->lpfc_drb_pool, dma_buf->dbuf.virt,
+ dma_buf->dbuf.phys);
+ pci_pool_free(phba->lpfc_hrb_pool, dma_buf->hbuf.virt,
+ dma_buf->hbuf.phys);
+ kfree(dma_buf);
+ lpfc_printf_log(phba, KERN_ERR, LOG_NVME,
+ "6132 Ran out of nvmet XRIs\n");
+ return NULL;
+ }
+ return dma_buf;
+}
+
+/**
+ * lpfc_sli4_nvmet_free - Frees a receive buffer
+ * @phba: HBA buffer was allocated for
+ * @dmab: DMA Buffer container returned by lpfc_sli4_rbq_alloc
+ *
+ * Description: Frees both the container and the DMA-mapped buffers returned by
+ * lpfc_sli4_nvmet_alloc.
+ *
+ * Notes: Can be called with or without locks held.
+ *
+ * Returns: None
+ **/
+void
+lpfc_sli4_nvmet_free(struct lpfc_hba *phba, struct rqb_dmabuf *dmab)
+{
+ unsigned long flags;
+
+ __lpfc_clear_active_sglq(phba, dmab->sglq->sli4_lxritag);
+ dmab->sglq->state = SGL_FREED;
+ dmab->sglq->ndlp = NULL;
+
+ spin_lock_irqsave(&phba->sli4_hba.sgl_list_lock, flags);
+ list_add_tail(&dmab->sglq->list, &phba->sli4_hba.lpfc_nvmet_sgl_list);
+ spin_unlock_irqrestore(&phba->sli4_hba.sgl_list_lock, flags);
+
+ lpfc_sli_release_iocbq(phba, dmab->iocbq);
+ kfree(dmab->context);
+ pci_pool_free(phba->lpfc_hrb_pool, dmab->hbuf.virt, dmab->hbuf.phys);
+ pci_pool_free(phba->lpfc_drb_pool, dmab->dbuf.virt, dmab->dbuf.phys);
+ kfree(dmab);
}
/**
return;
if (phba->sli3_options & LPFC_SLI3_HBQ_ENABLED) {
+ hbq_entry = container_of(mp, struct hbq_dmabuf, dbuf);
/* Check whether HBQ is still in use */
spin_lock_irqsave(&phba->hbalock, flags);
if (!phba->hbq_in_use) {
spin_unlock_irqrestore(&phba->hbalock, flags);
return;
}
- hbq_entry = container_of(mp, struct hbq_dmabuf, dbuf);
list_del(&hbq_entry->dbuf.list);
if (hbq_entry->tag == -1) {
(phba->hbqs[LPFC_ELS_HBQ].hbq_free_buffer)
}
return;
}
+
+/**
+ * lpfc_rq_buf_free - Free a RQ DMA buffer
+ * @phba: HBA buffer is associated with
+ * @mp: Buffer to free
+ *
+ * Description: Frees the given DMA buffer in the appropriate way given by
+ * reposting it to its associated RQ so it can be reused.
+ *
+ * Notes: Takes phba->hbalock. Can be called with or without other locks held.
+ *
+ * Returns: None
+ **/
+void
+lpfc_rq_buf_free(struct lpfc_hba *phba, struct lpfc_dmabuf *mp)
+{
+ struct lpfc_rqb *rqbp;
+ struct lpfc_rqe hrqe;
+ struct lpfc_rqe drqe;
+ struct rqb_dmabuf *rqb_entry;
+ unsigned long flags;
+ int rc;
+
+ if (!mp)
+ return;
+
+ rqb_entry = container_of(mp, struct rqb_dmabuf, hbuf);
+ rqbp = rqb_entry->hrq->rqbp;
+
+ spin_lock_irqsave(&phba->hbalock, flags);
+ list_del(&rqb_entry->hbuf.list);
+ hrqe.address_lo = putPaddrLow(rqb_entry->hbuf.phys);
+ hrqe.address_hi = putPaddrHigh(rqb_entry->hbuf.phys);
+ drqe.address_lo = putPaddrLow(rqb_entry->dbuf.phys);
+ drqe.address_hi = putPaddrHigh(rqb_entry->dbuf.phys);
+ rc = lpfc_sli4_rq_put(rqb_entry->hrq, rqb_entry->drq, &hrqe, &drqe);
+ if (rc < 0) {
+ (rqbp->rqb_free_buffer)(phba, rqb_entry);
+ } else {
+ list_add_tail(&rqb_entry->hbuf.list, &rqbp->rqb_buffer_list);
+ rqbp->buffer_count++;
+ }
+
+ spin_unlock_irqrestore(&phba->hbalock, flags);
+}
/*******************************************************************
* This file is part of the Emulex Linux Device Driver for *
* Fibre Channel Host Bus Adapters. *
+ * Copyright (C) 2017 Broadcom. All Rights Reserved. The term *
+ * “Broadcom” refers to Broadcom Limited and/or its subsidiaries. *
* Copyright (C) 2010 Emulex. All rights reserved. *
* EMULEX and SLI are trademarks of Emulex. *
- * www.emulex.com *
+ * www.broadcom.com *
* *
* This program is free software; you can redistribute it and/or *
* modify it under the terms of version 2 of the GNU General *
/*******************************************************************
* This file is part of the Emulex Linux Device Driver for *
* Fibre Channel Host Bus Adapters. *
+ * Copyright (C) 2017 Broadcom. All Rights Reserved. The term *
+ * “Broadcom” refers to Broadcom Limited and/or its subsidiaries. *
* Copyright (C) 2004-2016 Emulex. All rights reserved. *
* EMULEX and SLI are trademarks of Emulex. *
- * www.emulex.com *
+ * www.broadcom.com *
* Portions Copyright (C) 2004-2005 Christoph Hellwig *
* *
* This program is free software; you can redistribute it and/or *
#include <scsi/scsi_device.h>
#include <scsi/scsi_host.h>
#include <scsi/scsi_transport_fc.h>
+#include <scsi/fc/fc_fs.h>
+
+#include <linux/nvme-fc-driver.h>
#include "lpfc_hw4.h"
#include "lpfc_hw.h"
#include "lpfc_sli4.h"
#include "lpfc_nl.h"
#include "lpfc_disc.h"
-#include "lpfc_scsi.h"
#include "lpfc.h"
+#include "lpfc_scsi.h"
+#include "lpfc_nvme.h"
#include "lpfc_logmsg.h"
#include "lpfc_crtn.h"
#include "lpfc_vport.h"
lpfc_els_abort(struct lpfc_hba *phba, struct lpfc_nodelist *ndlp)
{
LIST_HEAD(abort_list);
- struct lpfc_sli *psli = &phba->sli;
- struct lpfc_sli_ring *pring = &psli->ring[LPFC_ELS_RING];
+ struct lpfc_sli_ring *pring;
struct lpfc_iocbq *iocb, *next_iocb;
+ pring = lpfc_phba_elsring(phba);
+
/* Abort outstanding I/O on NPort <nlp_DID> */
lpfc_printf_vlog(ndlp->vport, KERN_INFO, LOG_DISCOVERY,
"2819 Abort outstanding I/O on NPort x%x "
uint32_t ed_tov;
LPFC_MBOXQ_t *mbox;
struct ls_rjt stat;
+ uint32_t vid, flag;
int rc;
memset(&stat, 0, sizeof (struct ls_rjt));
lpfc_can_disctmo(vport);
}
+ ndlp->nlp_flag &= ~NLP_SUPPRESS_RSP;
+ if ((phba->sli.sli_flag & LPFC_SLI_SUPPRESS_RSP) &&
+ sp->cmn.valid_vendor_ver_level) {
+ vid = be32_to_cpu(sp->un.vv.vid);
+ flag = be32_to_cpu(sp->un.vv.flags);
+ if ((vid == LPFC_VV_EMLX_ID) && (flag & LPFC_VV_SUPPRESS_RSP))
+ ndlp->nlp_flag |= NLP_SUPPRESS_RSP;
+ }
+
mbox = mempool_alloc(phba->mbox_mem_pool, GFP_KERNEL);
if (!mbox)
goto out;
lpfc_rcv_prli(struct lpfc_vport *vport, struct lpfc_nodelist *ndlp,
struct lpfc_iocbq *cmdiocb)
{
+ struct lpfc_hba *phba = vport->phba;
struct lpfc_dmabuf *pcmd;
uint32_t *lp;
PRLI *npr;
ndlp->nlp_type &= ~(NLP_FCP_TARGET | NLP_FCP_INITIATOR);
ndlp->nlp_fcp_info &= ~NLP_FCP_2_DEVICE;
ndlp->nlp_flag &= ~NLP_FIRSTBURST;
- if (npr->prliType == PRLI_FCP_TYPE) {
- if (npr->initiatorFunc)
- ndlp->nlp_type |= NLP_FCP_INITIATOR;
+ if ((npr->prliType == PRLI_FCP_TYPE) ||
+ (npr->prliType == PRLI_NVME_TYPE)) {
+ if (npr->initiatorFunc) {
+ if (npr->prliType == PRLI_FCP_TYPE)
+ ndlp->nlp_type |= NLP_FCP_INITIATOR;
+ if (npr->prliType == PRLI_NVME_TYPE)
+ ndlp->nlp_type |= NLP_NVME_INITIATOR;
+ }
if (npr->targetFunc) {
- ndlp->nlp_type |= NLP_FCP_TARGET;
+ if (npr->prliType == PRLI_FCP_TYPE)
+ ndlp->nlp_type |= NLP_FCP_TARGET;
+ if (npr->prliType == PRLI_NVME_TYPE)
+ ndlp->nlp_type |= NLP_NVME_TARGET;
if (npr->writeXferRdyDis)
ndlp->nlp_flag |= NLP_FIRSTBURST;
}
if (npr->Retry)
ndlp->nlp_fcp_info |= NLP_FCP_2_DEVICE;
+
+ /* If this driver is in nvme target mode, set the ndlp's fc4
+ * type to NVME provided the PRLI response claims NVME FC4
+ * type. Target mode does not issue gft_id so doesn't get
+ * the fc4 type set until now.
+ */
+ if ((phba->nvmet_support) && (npr->prliType == PRLI_NVME_TYPE))
+ ndlp->nlp_fc4_type |= NLP_FC4_NVME;
}
if (rport) {
/* We need to update the rport role values */
"rport rolechg: role:x%x did:x%x flg:x%x",
roles, ndlp->nlp_DID, ndlp->nlp_flag);
- fc_remote_port_rolechg(rport, roles);
+ if (phba->cfg_enable_fc4_type != LPFC_ENABLE_NVME)
+ fc_remote_port_rolechg(rport, roles);
}
}
struct lpfc_iocbq *cmdiocb, *rspiocb;
struct lpfc_dmabuf *pcmd, *prsp, *mp;
uint32_t *lp;
+ uint32_t vid, flag;
IOCB_t *irsp;
struct serv_parm *sp;
uint32_t ed_tov;
ed_tov = (phba->fc_edtov + 999999) / 1000000;
}
+ ndlp->nlp_flag &= ~NLP_SUPPRESS_RSP;
+ if ((phba->sli.sli_flag & LPFC_SLI_SUPPRESS_RSP) &&
+ sp->cmn.valid_vendor_ver_level) {
+ vid = be32_to_cpu(sp->un.vv.vid);
+ flag = be32_to_cpu(sp->un.vv.flags);
+ if ((vid == LPFC_VV_EMLX_ID) &&
+ (flag & LPFC_VV_SUPPRESS_RSP))
+ ndlp->nlp_flag |= NLP_SUPPRESS_RSP;
+ }
+
/*
* Use the larger EDTOV
* RATOV = 2 * EDTOV for pt-to-pt
uint32_t evt)
{
struct lpfc_iocbq *cmdiocb = (struct lpfc_iocbq *) arg;
+ struct ls_rjt stat;
+
+ if (vport->phba->nvmet_support) {
+ /* NVME Target mode. Handle and respond to the PRLI and
+ * transition to UNMAPPED provided the RPI has completed
+ * registration.
+ */
+ if (ndlp->nlp_flag & NLP_RPI_REGISTERED) {
+ lpfc_rcv_prli(vport, ndlp, cmdiocb);
+ lpfc_els_rsp_prli_acc(vport, cmdiocb, ndlp);
+ lpfc_nlp_set_state(vport, ndlp, NLP_STE_UNMAPPED_NODE);
+ } else {
+ /* RPI registration has not completed. Reject the PRLI
+ * to prevent an illegal state transition when the
+ * rpi registration does complete.
+ */
+ lpfc_printf_vlog(vport, KERN_WARNING, LOG_NVME_DISC,
+ "6115 NVMET ndlp rpi %d state "
+ "unknown, state x%x flags x%08x\n",
+ ndlp->nlp_rpi, ndlp->nlp_state,
+ ndlp->nlp_flag);
+ memset(&stat, 0, sizeof(struct ls_rjt));
+ stat.un.b.lsRjtRsnCode = LSRJT_UNABLE_TPC;
+ stat.un.b.lsRjtRsnCodeExp = LSEXP_CMD_IN_PROGRESS;
+ lpfc_els_rsp_reject(vport, stat.un.lsRjtError, cmdiocb,
+ ndlp, NULL);
+ }
+ } else {
+ /* Initiator mode. */
+ lpfc_els_rsp_prli_acc(vport, cmdiocb, ndlp);
+ }
- lpfc_els_rsp_prli_acc(vport, cmdiocb, ndlp);
return ndlp->nlp_state;
}
uint32_t evt)
{
struct Scsi_Host *shost = lpfc_shost_from_vport(vport);
+ struct lpfc_hba *phba = vport->phba;
LPFC_MBOXQ_t *pmb = (LPFC_MBOXQ_t *) arg;
MAILBOX_t *mb = &pmb->u.mb;
uint32_t did = mb->un.varWords[1];
+ int rc = 0;
if (mb->mbxStatus) {
/* RegLogin failed */
}
/* SLI4 ports have preallocated logical rpis. */
- if (vport->phba->sli_rev < LPFC_SLI_REV4)
+ if (phba->sli_rev < LPFC_SLI_REV4)
ndlp->nlp_rpi = mb->un.varWords[0];
ndlp->nlp_flag |= NLP_RPI_REGISTERED;
/* Only if we are not a fabric nport do we issue PRLI */
- if (!(ndlp->nlp_type & NLP_FABRIC)) {
+ lpfc_printf_vlog(vport, KERN_INFO, LOG_DISCOVERY,
+ "3066 RegLogin Complete on x%x x%x x%x\n",
+ did, ndlp->nlp_type, ndlp->nlp_fc4_type);
+ if (!(ndlp->nlp_type & NLP_FABRIC) &&
+ (phba->nvmet_support == 0)) {
+ /* The driver supports FCP and NVME concurrently. If the
+ * ndlp's nlp_fc4_type is still zero, the driver doesn't
+ * know what PRLI to send yet. Figure that out now and
+ * call PRLI depending on the outcome.
+ */
+ if (vport->fc_flag & FC_PT2PT) {
+ /* If we are pt2pt, there is no Fabric to determine
+ * the FC4 type of the remote nport. So if NVME
+ * is configured try it.
+ */
+ ndlp->nlp_fc4_type |= NLP_FC4_FCP;
+ if ((phba->cfg_enable_fc4_type == LPFC_ENABLE_BOTH) ||
+ (phba->cfg_enable_fc4_type == LPFC_ENABLE_NVME)) {
+ ndlp->nlp_fc4_type |= NLP_FC4_NVME;
+ /* We need to update the localport also */
+ lpfc_nvme_update_localport(vport);
+ }
+
+ } else if (ndlp->nlp_fc4_type == 0) {
+ rc = lpfc_ns_cmd(vport, SLI_CTNS_GFT_ID,
+ 0, ndlp->nlp_DID);
+ return ndlp->nlp_state;
+ }
+
ndlp->nlp_prev_state = NLP_STE_REG_LOGIN_ISSUE;
lpfc_nlp_set_state(vport, ndlp, NLP_STE_PRLI_ISSUE);
lpfc_issue_els_prli(vport, ndlp, 0);
} else {
- ndlp->nlp_prev_state = NLP_STE_REG_LOGIN_ISSUE;
- lpfc_nlp_set_state(vport, ndlp, NLP_STE_UNMAPPED_NODE);
+ if ((vport->fc_flag & FC_PT2PT) && phba->nvmet_support)
+ phba->targetport->port_id = vport->fc_myDID;
+
+ /* Only Fabric ports should transition. NVME target
+ * must complete PRLI.
+ */
+ if (ndlp->nlp_type & NLP_FABRIC) {
+ ndlp->nlp_prev_state = NLP_STE_REG_LOGIN_ISSUE;
+ lpfc_nlp_set_state(vport, ndlp, NLP_STE_UNMAPPED_NODE);
+ }
}
return ndlp->nlp_state;
}
ndlp->nlp_prev_state = NLP_STE_REG_LOGIN_ISSUE;
lpfc_nlp_set_state(vport, ndlp, NLP_STE_NPR_NODE);
spin_lock_irq(shost->host_lock);
- ndlp->nlp_flag |= NLP_IGNR_REG_CMPL;
+
+ /* If we are a target we won't immediately transition into PRLI,
+ * so if REG_LOGIN already completed we don't need to ignore it.
+ */
+ if (!(ndlp->nlp_flag & NLP_RPI_REGISTERED) ||
+ !vport->phba->nvmet_support)
+ ndlp->nlp_flag |= NLP_IGNR_REG_CMPL;
+
ndlp->nlp_flag &= ~(NLP_NODEV_REMOVE | NLP_NPR_2B_DISC);
spin_unlock_irq(shost->host_lock);
lpfc_disc_set_adisc(vport, ndlp);
struct lpfc_hba *phba = vport->phba;
IOCB_t *irsp;
PRLI *npr;
+ struct lpfc_nvme_prli *nvpr;
+ void *temp_ptr;
cmdiocb = (struct lpfc_iocbq *) arg;
rspiocb = cmdiocb->context_un.rsp_iocb;
- npr = (PRLI *)lpfc_check_elscmpl_iocb(phba, cmdiocb, rspiocb);
+
+ /* A solicited PRLI is either FCP or NVME. The PRLI cmd/rsp
+ * format is different so NULL the two PRLI types so that the
+ * driver correctly gets the correct context.
+ */
+ npr = NULL;
+ nvpr = NULL;
+ temp_ptr = lpfc_check_elscmpl_iocb(phba, cmdiocb, rspiocb);
+ if (cmdiocb->iocb_flag & LPFC_PRLI_FCP_REQ)
+ npr = (PRLI *) temp_ptr;
+ else if (cmdiocb->iocb_flag & LPFC_PRLI_NVME_REQ)
+ nvpr = (struct lpfc_nvme_prli *) temp_ptr;
irsp = &rspiocb->iocb;
if (irsp->ulpStatus) {
vport->cfg_restrict_login) {
goto out;
}
+
+ /* The LS Req had some error. Don't let this be a
+ * target.
+ */
+ if ((ndlp->fc4_prli_sent == 1) &&
+ (ndlp->nlp_state == NLP_STE_PRLI_ISSUE) &&
+ (ndlp->nlp_type & (NLP_FCP_TARGET | NLP_FCP_INITIATOR)))
+ /* The FCP PRLI completed successfully but
+ * the NVME PRLI failed. Since they are sent in
+ * succession, allow the FCP to complete.
+ */
+ goto out_err;
+
ndlp->nlp_prev_state = NLP_STE_PRLI_ISSUE;
+ ndlp->nlp_type |= NLP_FCP_INITIATOR;
lpfc_nlp_set_state(vport, ndlp, NLP_STE_UNMAPPED_NODE);
return ndlp->nlp_state;
}
/* Check out PRLI rsp */
ndlp->nlp_type &= ~(NLP_FCP_TARGET | NLP_FCP_INITIATOR);
ndlp->nlp_fcp_info &= ~NLP_FCP_2_DEVICE;
+
+ /* NVME or FCP first burst must be negotiated for each PRLI. */
ndlp->nlp_flag &= ~NLP_FIRSTBURST;
- if ((npr->acceptRspCode == PRLI_REQ_EXECUTED) &&
+ ndlp->nvme_fb_size = 0;
+ if (npr && (npr->acceptRspCode == PRLI_REQ_EXECUTED) &&
(npr->prliType == PRLI_FCP_TYPE)) {
+ lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_DISC,
+ "6028 FCP NPR PRLI Cmpl Init %d Target %d\n",
+ npr->initiatorFunc,
+ npr->targetFunc);
if (npr->initiatorFunc)
ndlp->nlp_type |= NLP_FCP_INITIATOR;
if (npr->targetFunc) {
}
if (npr->Retry)
ndlp->nlp_fcp_info |= NLP_FCP_2_DEVICE;
+
+ /* PRLI completed. Decrement count. */
+ ndlp->fc4_prli_sent--;
+ } else if (nvpr &&
+ (bf_get_be32(prli_acc_rsp_code, nvpr) ==
+ PRLI_REQ_EXECUTED) &&
+ (bf_get_be32(prli_type_code, nvpr) ==
+ PRLI_NVME_TYPE)) {
+
+ /* Complete setting up the remote ndlp personality. */
+ if (bf_get_be32(prli_init, nvpr))
+ ndlp->nlp_type |= NLP_NVME_INITIATOR;
+
+ /* Target driver cannot solicit NVME FB. */
+ if (bf_get_be32(prli_tgt, nvpr)) {
+ ndlp->nlp_type |= NLP_NVME_TARGET;
+ if ((bf_get_be32(prli_fba, nvpr) == 1) &&
+ (bf_get_be32(prli_fb_sz, nvpr) > 0) &&
+ (phba->cfg_nvme_enable_fb) &&
+ (!phba->nvmet_support)) {
+ /* Both sides support FB. The target's first
+ * burst size is a 512 byte encoded value.
+ */
+ ndlp->nlp_flag |= NLP_FIRSTBURST;
+ ndlp->nvme_fb_size = bf_get_be32(prli_fb_sz,
+ nvpr);
+ }
+ }
+
+ if (bf_get_be32(prli_recov, nvpr))
+ ndlp->nlp_fcp_info |= NLP_FCP_2_DEVICE;
+
+ lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_DISC,
+ "6029 NVME PRLI Cmpl w1 x%08x "
+ "w4 x%08x w5 x%08x flag x%x, "
+ "fcp_info x%x nlp_type x%x\n",
+ be32_to_cpu(nvpr->word1),
+ be32_to_cpu(nvpr->word4),
+ be32_to_cpu(nvpr->word5),
+ ndlp->nlp_flag, ndlp->nlp_fcp_info,
+ ndlp->nlp_type);
+ /* PRLI completed. Decrement count. */
+ ndlp->fc4_prli_sent--;
}
if (!(ndlp->nlp_type & NLP_FCP_TARGET) &&
(vport->port_type == LPFC_NPIV_PORT) &&
return ndlp->nlp_state;
}
- ndlp->nlp_prev_state = NLP_STE_PRLI_ISSUE;
- if (ndlp->nlp_type & NLP_FCP_TARGET)
- lpfc_nlp_set_state(vport, ndlp, NLP_STE_MAPPED_NODE);
- else
- lpfc_nlp_set_state(vport, ndlp, NLP_STE_UNMAPPED_NODE);
+out_err:
+ /* The ndlp state cannot move to MAPPED or UNMAPPED before all PRLIs
+ * are complete.
+ */
+ if (ndlp->fc4_prli_sent == 0) {
+ ndlp->nlp_prev_state = NLP_STE_PRLI_ISSUE;
+ if (ndlp->nlp_type & (NLP_FCP_TARGET | NLP_NVME_TARGET))
+ lpfc_nlp_set_state(vport, ndlp, NLP_STE_MAPPED_NODE);
+ else
+ lpfc_nlp_set_state(vport, ndlp, NLP_STE_UNMAPPED_NODE);
+ } else
+ lpfc_printf_vlog(vport,
+ KERN_INFO, LOG_ELS,
+ "3067 PRLI's still outstanding "
+ "on x%06x - count %d, Pend Node Mode "
+ "transition...\n",
+ ndlp->nlp_DID, ndlp->fc4_prli_sent);
+
return ndlp->nlp_state;
}
struct lpfc_iocbq *cmdiocb = (struct lpfc_iocbq *) arg;
/* flush the target */
- lpfc_sli_abort_iocb(vport, &phba->sli.ring[phba->sli.fcp_ring],
+ lpfc_sli_abort_iocb(vport, &phba->sli.sli3_ring[LPFC_FCP_RING],
ndlp->nlp_sid, 0, LPFC_CTX_TGT);
/* Treat like rcv logo */
--- /dev/null
+/*******************************************************************
+ * This file is part of the Emulex Linux Device Driver for *
+ * Fibre Channel Host Bus Adapters. *
+ * Copyright (C) 2017 Broadcom. All Rights Reserved. The term *
+ * “Broadcom” refers to Broadcom Limited and/or its subsidiaries. *
+ * Copyright (C) 2004-2016 Emulex. All rights reserved. *
+ * EMULEX and SLI are trademarks of Emulex. *
+ * www.broadcom.com *
+ * Portions Copyright (C) 2004-2005 Christoph Hellwig *
+ * *
+ * This program is free software; you can redistribute it and/or *
+ * modify it under the terms of version 2 of the GNU General *
+ * Public License as published by the Free Software Foundation. *
+ * This program is distributed in the hope that it will be useful. *
+ * ALL EXPRESS OR IMPLIED CONDITIONS, REPRESENTATIONS AND *
+ * WARRANTIES, INCLUDING ANY IMPLIED WARRANTY OF MERCHANTABILITY, *
+ * FITNESS FOR A PARTICULAR PURPOSE, OR NON-INFRINGEMENT, ARE *
+ * DISCLAIMED, EXCEPT TO THE EXTENT THAT SUCH DISCLAIMERS ARE HELD *
+ * TO BE LEGALLY INVALID. See the GNU General Public License for *
+ * more details, a copy of which can be found in the file COPYING *
+ * included with this package. *
+ ********************************************************************/
+#include <linux/pci.h>
+#include <linux/slab.h>
+#include <linux/interrupt.h>
+#include <linux/delay.h>
+#include <asm/unaligned.h>
+#include <linux/crc-t10dif.h>
+#include <net/checksum.h>
+
+#include <scsi/scsi.h>
+#include <scsi/scsi_device.h>
+#include <scsi/scsi_eh.h>
+#include <scsi/scsi_host.h>
+#include <scsi/scsi_tcq.h>
+#include <scsi/scsi_transport_fc.h>
+#include <scsi/fc/fc_fs.h>
+
+#include <linux/nvme.h>
+#include <linux/nvme-fc-driver.h>
+#include <linux/nvme-fc.h>
+#include "lpfc_version.h"
+#include "lpfc_hw4.h"
+#include "lpfc_hw.h"
+#include "lpfc_sli.h"
+#include "lpfc_sli4.h"
+#include "lpfc_nl.h"
+#include "lpfc_disc.h"
+#include "lpfc.h"
+#include "lpfc_nvme.h"
+#include "lpfc_scsi.h"
+#include "lpfc_logmsg.h"
+#include "lpfc_crtn.h"
+#include "lpfc_vport.h"
+#include "lpfc_debugfs.h"
+
+/* NVME initiator-based functions */
+
+static struct lpfc_nvme_buf *
+lpfc_get_nvme_buf(struct lpfc_hba *phba, struct lpfc_nodelist *ndlp);
+
+static void
+lpfc_release_nvme_buf(struct lpfc_hba *, struct lpfc_nvme_buf *);
+
+
+/**
+ * lpfc_nvme_create_queue -
+ * @lpfc_pnvme: Pointer to the driver's nvme instance data
+ * @qidx: An cpu index used to affinitize IO queues and MSIX vectors.
+ * @handle: An opaque driver handle used in follow-up calls.
+ *
+ * Driver registers this routine to preallocate and initialize any
+ * internal data structures to bind the @qidx to its internal IO queues.
+ * A hardware queue maps (qidx) to a specific driver MSI-X vector/EQ/CQ/WQ.
+ *
+ * Return value :
+ * 0 - Success
+ * -EINVAL - Unsupported input value.
+ * -ENOMEM - Could not alloc necessary memory
+ **/
+static int
+lpfc_nvme_create_queue(struct nvme_fc_local_port *pnvme_lport,
+ unsigned int qidx, u16 qsize,
+ void **handle)
+{
+ struct lpfc_nvme_lport *lport;
+ struct lpfc_vport *vport;
+ struct lpfc_nvme_qhandle *qhandle;
+ char *str;
+
+ lport = (struct lpfc_nvme_lport *)pnvme_lport->private;
+ vport = lport->vport;
+ qhandle = kzalloc(sizeof(struct lpfc_nvme_qhandle), GFP_KERNEL);
+ if (qhandle == NULL)
+ return -ENOMEM;
+
+ qhandle->cpu_id = smp_processor_id();
+ qhandle->qidx = qidx;
+ /*
+ * NVME qidx == 0 is the admin queue, so both admin queue
+ * and first IO queue will use MSI-X vector and associated
+ * EQ/CQ/WQ at index 0. After that they are sequentially assigned.
+ */
+ if (qidx) {
+ str = "IO "; /* IO queue */
+ qhandle->index = ((qidx - 1) %
+ vport->phba->cfg_nvme_io_channel);
+ } else {
+ str = "ADM"; /* Admin queue */
+ qhandle->index = qidx;
+ }
+
+ lpfc_printf_vlog(vport, KERN_ERR, LOG_NVME,
+ "6073 Binding %s HdwQueue %d (cpu %d) to "
+ "io_channel %d qhandle %p\n", str,
+ qidx, qhandle->cpu_id, qhandle->index, qhandle);
+ *handle = (void *)qhandle;
+ return 0;
+}
+
+/**
+ * lpfc_nvme_delete_queue -
+ * @lpfc_pnvme: Pointer to the driver's nvme instance data
+ * @qidx: An cpu index used to affinitize IO queues and MSIX vectors.
+ * @handle: An opaque driver handle from lpfc_nvme_create_queue
+ *
+ * Driver registers this routine to free
+ * any internal data structures to bind the @qidx to its internal
+ * IO queues.
+ *
+ * Return value :
+ * 0 - Success
+ * TODO: What are the failure codes.
+ **/
+static void
+lpfc_nvme_delete_queue(struct nvme_fc_local_port *pnvme_lport,
+ unsigned int qidx,
+ void *handle)
+{
+ struct lpfc_nvme_lport *lport;
+ struct lpfc_vport *vport;
+
+ lport = (struct lpfc_nvme_lport *)pnvme_lport->private;
+ vport = lport->vport;
+
+ lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME,
+ "6001 ENTER. lpfc_pnvme %p, qidx x%xi qhandle %p\n",
+ lport, qidx, handle);
+ kfree(handle);
+}
+
+static void
+lpfc_nvme_localport_delete(struct nvme_fc_local_port *localport)
+{
+ struct lpfc_nvme_lport *lport = localport->private;
+
+ /* release any threads waiting for the unreg to complete */
+ complete(&lport->lport_unreg_done);
+}
+
+/* lpfc_nvme_remoteport_delete
+ *
+ * @remoteport: Pointer to an nvme transport remoteport instance.
+ *
+ * This is a template downcall. NVME transport calls this function
+ * when it has completed the unregistration of a previously
+ * registered remoteport.
+ *
+ * Return value :
+ * None
+ */
+void
+lpfc_nvme_remoteport_delete(struct nvme_fc_remote_port *remoteport)
+{
+ struct lpfc_nvme_rport *rport = remoteport->private;
+ struct lpfc_vport *vport;
+ struct lpfc_nodelist *ndlp;
+
+ ndlp = rport->ndlp;
+ if (!ndlp)
+ goto rport_err;
+
+ vport = ndlp->vport;
+ if (!vport)
+ goto rport_err;
+
+ /* Remove this rport from the lport's list - memory is owned by the
+ * transport. Remove the ndlp reference for the NVME transport before
+ * calling state machine to remove the node, this is devloss = 0
+ * semantics.
+ */
+ lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_DISC,
+ "6146 remoteport delete complete %p\n",
+ remoteport);
+ list_del(&rport->list);
+ lpfc_nlp_put(ndlp);
+
+ rport_err:
+ /* This call has to execute as long as the rport is valid.
+ * Release any threads waiting for the unreg to complete.
+ */
+ complete(&rport->rport_unreg_done);
+}
+
+static void
+lpfc_nvme_cmpl_gen_req(struct lpfc_hba *phba, struct lpfc_iocbq *cmdwqe,
+ struct lpfc_wcqe_complete *wcqe)
+{
+ struct lpfc_vport *vport = cmdwqe->vport;
+ uint32_t status;
+ struct nvmefc_ls_req *pnvme_lsreq;
+ struct lpfc_dmabuf *buf_ptr;
+ struct lpfc_nodelist *ndlp;
+
+ vport->phba->fc4NvmeLsCmpls++;
+
+ pnvme_lsreq = (struct nvmefc_ls_req *)cmdwqe->context2;
+ status = bf_get(lpfc_wcqe_c_status, wcqe) & LPFC_IOCB_STATUS_MASK;
+ ndlp = (struct lpfc_nodelist *)cmdwqe->context1;
+ lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_DISC,
+ "6047 nvme cmpl Enter "
+ "Data %p DID %x Xri: %x status %x cmd:%p lsreg:%p "
+ "bmp:%p ndlp:%p\n",
+ pnvme_lsreq, ndlp ? ndlp->nlp_DID : 0,
+ cmdwqe->sli4_xritag, status,
+ cmdwqe, pnvme_lsreq, cmdwqe->context3, ndlp);
+
+ lpfc_nvmeio_data(phba, "NVME LS CMPL: xri x%x stat x%x parm x%x\n",
+ cmdwqe->sli4_xritag, status, wcqe->parameter);
+
+ if (cmdwqe->context3) {
+ buf_ptr = (struct lpfc_dmabuf *)cmdwqe->context3;
+ lpfc_mbuf_free(phba, buf_ptr->virt, buf_ptr->phys);
+ kfree(buf_ptr);
+ cmdwqe->context3 = NULL;
+ }
+ if (pnvme_lsreq->done)
+ pnvme_lsreq->done(pnvme_lsreq, status);
+ else
+ lpfc_printf_vlog(vport, KERN_ERR, LOG_NVME_DISC,
+ "6046 nvme cmpl without done call back? "
+ "Data %p DID %x Xri: %x status %x\n",
+ pnvme_lsreq, ndlp ? ndlp->nlp_DID : 0,
+ cmdwqe->sli4_xritag, status);
+ if (ndlp) {
+ lpfc_nlp_put(ndlp);
+ cmdwqe->context1 = NULL;
+ }
+ lpfc_sli_release_iocbq(phba, cmdwqe);
+}
+
+static int
+lpfc_nvme_gen_req(struct lpfc_vport *vport, struct lpfc_dmabuf *bmp,
+ struct lpfc_dmabuf *inp,
+ struct nvmefc_ls_req *pnvme_lsreq,
+ void (*cmpl)(struct lpfc_hba *, struct lpfc_iocbq *,
+ struct lpfc_wcqe_complete *),
+ struct lpfc_nodelist *ndlp, uint32_t num_entry,
+ uint32_t tmo, uint8_t retry)
+{
+ struct lpfc_hba *phba = vport->phba;
+ union lpfc_wqe *wqe;
+ struct lpfc_iocbq *genwqe;
+ struct ulp_bde64 *bpl;
+ struct ulp_bde64 bde;
+ int i, rc, xmit_len, first_len;
+
+ /* Allocate buffer for command WQE */
+ genwqe = lpfc_sli_get_iocbq(phba);
+ if (genwqe == NULL)
+ return 1;
+
+ wqe = &genwqe->wqe;
+ memset(wqe, 0, sizeof(union lpfc_wqe));
+
+ genwqe->context3 = (uint8_t *)bmp;
+ genwqe->iocb_flag |= LPFC_IO_NVME_LS;
+
+ /* Save for completion so we can release these resources */
+ genwqe->context1 = lpfc_nlp_get(ndlp);
+ genwqe->context2 = (uint8_t *)pnvme_lsreq;
+ /* Fill in payload, bp points to frame payload */
+
+ if (!tmo)
+ /* FC spec states we need 3 * ratov for CT requests */
+ tmo = (3 * phba->fc_ratov);
+
+ /* For this command calculate the xmit length of the request bde. */
+ xmit_len = 0;
+ first_len = 0;
+ bpl = (struct ulp_bde64 *)bmp->virt;
+ for (i = 0; i < num_entry; i++) {
+ bde.tus.w = bpl[i].tus.w;
+ if (bde.tus.f.bdeFlags != BUFF_TYPE_BDE_64)
+ break;
+ xmit_len += bde.tus.f.bdeSize;
+ if (i == 0)
+ first_len = xmit_len;
+ }
+
+ genwqe->rsvd2 = num_entry;
+ genwqe->hba_wqidx = 0;
+
+ /* Words 0 - 2 */
+ wqe->generic.bde.tus.f.bdeFlags = BUFF_TYPE_BDE_64;
+ wqe->generic.bde.tus.f.bdeSize = first_len;
+ wqe->generic.bde.addrLow = bpl[0].addrLow;
+ wqe->generic.bde.addrHigh = bpl[0].addrHigh;
+
+ /* Word 3 */
+ wqe->gen_req.request_payload_len = first_len;
+
+ /* Word 4 */
+
+ /* Word 5 */
+ bf_set(wqe_dfctl, &wqe->gen_req.wge_ctl, 0);
+ bf_set(wqe_si, &wqe->gen_req.wge_ctl, 1);
+ bf_set(wqe_la, &wqe->gen_req.wge_ctl, 1);
+ bf_set(wqe_rctl, &wqe->gen_req.wge_ctl, FC_RCTL_DD_UNSOL_CTL);
+ bf_set(wqe_type, &wqe->gen_req.wge_ctl, FC_TYPE_NVME);
+
+ /* Word 6 */
+ bf_set(wqe_ctxt_tag, &wqe->gen_req.wqe_com,
+ phba->sli4_hba.rpi_ids[ndlp->nlp_rpi]);
+ bf_set(wqe_xri_tag, &wqe->gen_req.wqe_com, genwqe->sli4_xritag);
+
+ /* Word 7 */
+ bf_set(wqe_tmo, &wqe->gen_req.wqe_com, (vport->phba->fc_ratov-1));
+ bf_set(wqe_class, &wqe->gen_req.wqe_com, CLASS3);
+ bf_set(wqe_cmnd, &wqe->gen_req.wqe_com, CMD_GEN_REQUEST64_WQE);
+ bf_set(wqe_ct, &wqe->gen_req.wqe_com, SLI4_CT_RPI);
+
+ /* Word 8 */
+ wqe->gen_req.wqe_com.abort_tag = genwqe->iotag;
+
+ /* Word 9 */
+ bf_set(wqe_reqtag, &wqe->gen_req.wqe_com, genwqe->iotag);
+
+ /* Word 10 */
+ bf_set(wqe_dbde, &wqe->gen_req.wqe_com, 1);
+ bf_set(wqe_iod, &wqe->gen_req.wqe_com, LPFC_WQE_IOD_READ);
+ bf_set(wqe_qosd, &wqe->gen_req.wqe_com, 1);
+ bf_set(wqe_lenloc, &wqe->gen_req.wqe_com, LPFC_WQE_LENLOC_NONE);
+ bf_set(wqe_ebde_cnt, &wqe->gen_req.wqe_com, 0);
+
+ /* Word 11 */
+ bf_set(wqe_cqid, &wqe->gen_req.wqe_com, LPFC_WQE_CQ_ID_DEFAULT);
+ bf_set(wqe_cmd_type, &wqe->gen_req.wqe_com, OTHER_COMMAND);
+
+
+ /* Issue GEN REQ WQE for NPORT <did> */
+ lpfc_printf_vlog(vport, KERN_INFO, LOG_ELS,
+ "6050 Issue GEN REQ WQE to NPORT x%x "
+ "Data: x%x x%x wq:%p lsreq:%p bmp:%p xmit:%d 1st:%d\n",
+ ndlp->nlp_DID, genwqe->iotag,
+ vport->port_state,
+ genwqe, pnvme_lsreq, bmp, xmit_len, first_len);
+ genwqe->wqe_cmpl = cmpl;
+ genwqe->iocb_cmpl = NULL;
+ genwqe->drvrTimeout = tmo + LPFC_DRVR_TIMEOUT;
+ genwqe->vport = vport;
+ genwqe->retry = retry;
+
+ lpfc_nvmeio_data(phba, "NVME LS XMIT: xri x%x iotag x%x to x%06x\n",
+ genwqe->sli4_xritag, genwqe->iotag, ndlp->nlp_DID);
+
+ rc = lpfc_sli4_issue_wqe(phba, LPFC_ELS_RING, genwqe);
+ if (rc == WQE_ERROR) {
+ lpfc_printf_vlog(vport, KERN_ERR, LOG_ELS,
+ "6045 Issue GEN REQ WQE to NPORT x%x "
+ "Data: x%x x%x\n",
+ ndlp->nlp_DID, genwqe->iotag,
+ vport->port_state);
+ lpfc_sli_release_iocbq(phba, genwqe);
+ return 1;
+ }
+ return 0;
+}
+
+/**
+ * lpfc_nvme_ls_req - Issue an Link Service request
+ * @lpfc_pnvme: Pointer to the driver's nvme instance data
+ * @lpfc_nvme_lport: Pointer to the driver's local port data
+ * @lpfc_nvme_rport: Pointer to the rport getting the @lpfc_nvme_ereq
+ *
+ * Driver registers this routine to handle any link service request
+ * from the nvme_fc transport to a remote nvme-aware port.
+ *
+ * Return value :
+ * 0 - Success
+ * TODO: What are the failure codes.
+ **/
+static int
+lpfc_nvme_ls_req(struct nvme_fc_local_port *pnvme_lport,
+ struct nvme_fc_remote_port *pnvme_rport,
+ struct nvmefc_ls_req *pnvme_lsreq)
+{
+ int ret = 0;
+ struct lpfc_nvme_lport *lport;
+ struct lpfc_vport *vport;
+ struct lpfc_nodelist *ndlp;
+ struct ulp_bde64 *bpl;
+ struct lpfc_dmabuf *bmp;
+
+ /* there are two dma buf in the request, actually there is one and
+ * the second one is just the start address + cmd size.
+ * Before calling lpfc_nvme_gen_req these buffers need to be wrapped
+ * in a lpfc_dmabuf struct. When freeing we just free the wrapper
+ * because the nvem layer owns the data bufs.
+ * We do not have to break these packets open, we don't care what is in
+ * them. And we do not have to look at the resonse data, we only care
+ * that we got a response. All of the caring is going to happen in the
+ * nvme-fc layer.
+ */
+
+ lport = (struct lpfc_nvme_lport *)pnvme_lport->private;
+ vport = lport->vport;
+
+ ndlp = lpfc_findnode_did(vport, pnvme_rport->port_id);
+ if (!ndlp) {
+ lpfc_printf_vlog(vport, KERN_ERR, LOG_NVME_DISC,
+ "6043 Could not find node for DID %x\n",
+ pnvme_rport->port_id);
+ return 1;
+ }
+ bmp = kmalloc(sizeof(struct lpfc_dmabuf), GFP_KERNEL);
+ if (!bmp) {
+
+ lpfc_printf_vlog(vport, KERN_ERR, LOG_NVME_DISC,
+ "6044 Could not find node for DID %x\n",
+ pnvme_rport->port_id);
+ return 2;
+ }
+ INIT_LIST_HEAD(&bmp->list);
+ bmp->virt = lpfc_mbuf_alloc(vport->phba, MEM_PRI, &(bmp->phys));
+ if (!bmp->virt) {
+ lpfc_printf_vlog(vport, KERN_ERR, LOG_NVME_DISC,
+ "6042 Could not find node for DID %x\n",
+ pnvme_rport->port_id);
+ kfree(bmp);
+ return 3;
+ }
+ bpl = (struct ulp_bde64 *)bmp->virt;
+ bpl->addrHigh = le32_to_cpu(putPaddrHigh(pnvme_lsreq->rqstdma));
+ bpl->addrLow = le32_to_cpu(putPaddrLow(pnvme_lsreq->rqstdma));
+ bpl->tus.f.bdeFlags = 0;
+ bpl->tus.f.bdeSize = pnvme_lsreq->rqstlen;
+ bpl->tus.w = le32_to_cpu(bpl->tus.w);
+ bpl++;
+
+ bpl->addrHigh = le32_to_cpu(putPaddrHigh(pnvme_lsreq->rspdma));
+ bpl->addrLow = le32_to_cpu(putPaddrLow(pnvme_lsreq->rspdma));
+ bpl->tus.f.bdeFlags = BUFF_TYPE_BDE_64I;
+ bpl->tus.f.bdeSize = pnvme_lsreq->rsplen;
+ bpl->tus.w = le32_to_cpu(bpl->tus.w);
+
+ /* Expand print to include key fields. */
+ lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_DISC,
+ "6051 ENTER. lport %p, rport %p lsreq%p rqstlen:%d "
+ "rsplen:%d %pad %pad\n",
+ pnvme_lport, pnvme_rport,
+ pnvme_lsreq, pnvme_lsreq->rqstlen,
+ pnvme_lsreq->rsplen, &pnvme_lsreq->rqstdma,
+ &pnvme_lsreq->rspdma);
+
+ vport->phba->fc4NvmeLsRequests++;
+
+ /* Hardcode the wait to 30 seconds. Connections are failing otherwise.
+ * This code allows it all to work.
+ */
+ ret = lpfc_nvme_gen_req(vport, bmp, pnvme_lsreq->rqstaddr,
+ pnvme_lsreq, lpfc_nvme_cmpl_gen_req,
+ ndlp, 2, 30, 0);
+ if (ret != WQE_SUCCESS) {
+ lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_DISC,
+ "6052 EXIT. issue ls wqe failed lport %p, "
+ "rport %p lsreq%p Status %x DID %x\n",
+ pnvme_lport, pnvme_rport, pnvme_lsreq,
+ ret, ndlp->nlp_DID);
+ lpfc_mbuf_free(vport->phba, bmp->virt, bmp->phys);
+ kfree(bmp);
+ return ret;
+ }
+
+ /* Stub in routine and return 0 for now. */
+ return ret;
+}
+
+/**
+ * lpfc_nvme_ls_abort - Issue an Link Service request
+ * @lpfc_pnvme: Pointer to the driver's nvme instance data
+ * @lpfc_nvme_lport: Pointer to the driver's local port data
+ * @lpfc_nvme_rport: Pointer to the rport getting the @lpfc_nvme_ereq
+ *
+ * Driver registers this routine to handle any link service request
+ * from the nvme_fc transport to a remote nvme-aware port.
+ *
+ * Return value :
+ * 0 - Success
+ * TODO: What are the failure codes.
+ **/
+static void
+lpfc_nvme_ls_abort(struct nvme_fc_local_port *pnvme_lport,
+ struct nvme_fc_remote_port *pnvme_rport,
+ struct nvmefc_ls_req *pnvme_lsreq)
+{
+ struct lpfc_nvme_lport *lport;
+ struct lpfc_vport *vport;
+ struct lpfc_hba *phba;
+ struct lpfc_nodelist *ndlp;
+ LIST_HEAD(abort_list);
+ struct lpfc_sli_ring *pring;
+ struct lpfc_iocbq *wqe, *next_wqe;
+
+ lport = (struct lpfc_nvme_lport *)pnvme_lport->private;
+ vport = lport->vport;
+ phba = vport->phba;
+
+ ndlp = lpfc_findnode_did(vport, pnvme_rport->port_id);
+ if (!ndlp) {
+ lpfc_printf_vlog(vport, KERN_ERR, LOG_NVME_ABTS,
+ "6049 Could not find node for DID %x\n",
+ pnvme_rport->port_id);
+ return;
+ }
+
+ /* Expand print to include key fields. */
+ lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_ABTS,
+ "6040 ENTER. lport %p, rport %p lsreq %p rqstlen:%d "
+ "rsplen:%d %pad %pad\n",
+ pnvme_lport, pnvme_rport,
+ pnvme_lsreq, pnvme_lsreq->rqstlen,
+ pnvme_lsreq->rsplen, &pnvme_lsreq->rqstdma,
+ &pnvme_lsreq->rspdma);
+
+ /*
+ * Lock the ELS ring txcmplq and build a local list of all ELS IOs
+ * that need an ABTS. The IOs need to stay on the txcmplq so that
+ * the abort operation completes them successfully.
+ */
+ pring = phba->sli4_hba.nvmels_wq->pring;
+ spin_lock_irq(&phba->hbalock);
+ spin_lock(&pring->ring_lock);
+ list_for_each_entry_safe(wqe, next_wqe, &pring->txcmplq, list) {
+ /* Add to abort_list on on NDLP match. */
+ if (lpfc_check_sli_ndlp(phba, pring, wqe, ndlp)) {
+ wqe->iocb_flag |= LPFC_DRIVER_ABORTED;
+ list_add_tail(&wqe->dlist, &abort_list);
+ }
+ }
+ spin_unlock(&pring->ring_lock);
+ spin_unlock_irq(&phba->hbalock);
+
+ /* Abort the targeted IOs and remove them from the abort list. */
+ list_for_each_entry_safe(wqe, next_wqe, &abort_list, dlist) {
+ spin_lock_irq(&phba->hbalock);
+ list_del_init(&wqe->dlist);
+ lpfc_sli_issue_abort_iotag(phba, pring, wqe);
+ spin_unlock_irq(&phba->hbalock);
+ }
+}
+
+/* Fix up the existing sgls for NVME IO. */
+static void
+lpfc_nvme_adj_fcp_sgls(struct lpfc_vport *vport,
+ struct lpfc_nvme_buf *lpfc_ncmd,
+ struct nvmefc_fcp_req *nCmd)
+{
+ struct sli4_sge *sgl;
+ union lpfc_wqe128 *wqe;
+ uint32_t *wptr, *dptr;
+
+ /*
+ * Adjust the FCP_CMD and FCP_RSP DMA data and sge_len to
+ * match NVME. NVME sends 96 bytes. Also, use the
+ * nvme commands command and response dma addresses
+ * rather than the virtual memory to ease the restore
+ * operation.
+ */
+ sgl = lpfc_ncmd->nvme_sgl;
+ sgl->sge_len = cpu_to_le32(nCmd->cmdlen);
+
+ sgl++;
+
+ /* Setup the physical region for the FCP RSP */
+ sgl->addr_hi = cpu_to_le32(putPaddrHigh(nCmd->rspdma));
+ sgl->addr_lo = cpu_to_le32(putPaddrLow(nCmd->rspdma));
+ sgl->word2 = le32_to_cpu(sgl->word2);
+ if (nCmd->sg_cnt)
+ bf_set(lpfc_sli4_sge_last, sgl, 0);
+ else
+ bf_set(lpfc_sli4_sge_last, sgl, 1);
+ sgl->word2 = cpu_to_le32(sgl->word2);
+ sgl->sge_len = cpu_to_le32(nCmd->rsplen);
+
+ /*
+ * Get a local pointer to the built-in wqe and correct
+ * the cmd size to match NVME's 96 bytes and fix
+ * the dma address.
+ */
+
+ /* 128 byte wqe support here */
+ wqe = (union lpfc_wqe128 *)&lpfc_ncmd->cur_iocbq.wqe;
+
+ /* Word 0-2 - NVME CMND IU (embedded payload) */
+ wqe->generic.bde.tus.f.bdeFlags = BUFF_TYPE_BDE_IMMED;
+ wqe->generic.bde.tus.f.bdeSize = 60;
+ wqe->generic.bde.addrHigh = 0;
+ wqe->generic.bde.addrLow = 64; /* Word 16 */
+
+ /* Word 3 */
+ bf_set(payload_offset_len, &wqe->fcp_icmd,
+ (nCmd->rsplen + nCmd->cmdlen));
+
+ /* Word 10 */
+ bf_set(wqe_nvme, &wqe->fcp_icmd.wqe_com, 1);
+ bf_set(wqe_wqes, &wqe->fcp_icmd.wqe_com, 1);
+
+ /*
+ * Embed the payload in the last half of the WQE
+ * WQE words 16-30 get the NVME CMD IU payload
+ *
+ * WQE Word 16 is already setup with flags
+ * WQE words 17-19 get payload Words 2-4
+ * WQE words 20-21 get payload Words 6-7
+ * WQE words 22-29 get payload Words 16-23
+ */
+ wptr = &wqe->words[17]; /* WQE ptr */
+ dptr = (uint32_t *)nCmd->cmdaddr; /* payload ptr */
+ dptr += 2; /* Skip Words 0-1 in payload */
+
+ *wptr++ = *dptr++; /* Word 2 */
+ *wptr++ = *dptr++; /* Word 3 */
+ *wptr++ = *dptr++; /* Word 4 */
+ dptr++; /* Skip Word 5 in payload */
+ *wptr++ = *dptr++; /* Word 6 */
+ *wptr++ = *dptr++; /* Word 7 */
+ dptr += 8; /* Skip Words 8-15 in payload */
+ *wptr++ = *dptr++; /* Word 16 */
+ *wptr++ = *dptr++; /* Word 17 */
+ *wptr++ = *dptr++; /* Word 18 */
+ *wptr++ = *dptr++; /* Word 19 */
+ *wptr++ = *dptr++; /* Word 20 */
+ *wptr++ = *dptr++; /* Word 21 */
+ *wptr++ = *dptr++; /* Word 22 */
+ *wptr = *dptr; /* Word 23 */
+}
+
+#ifdef CONFIG_SCSI_LPFC_DEBUG_FS
+static void
+lpfc_nvme_ktime(struct lpfc_hba *phba,
+ struct lpfc_nvme_buf *lpfc_ncmd)
+{
+ uint64_t seg1, seg2, seg3, seg4;
+
+ if (!phba->ktime_on)
+ return;
+ if (!lpfc_ncmd->ts_last_cmd ||
+ !lpfc_ncmd->ts_cmd_start ||
+ !lpfc_ncmd->ts_cmd_wqput ||
+ !lpfc_ncmd->ts_isr_cmpl ||
+ !lpfc_ncmd->ts_data_nvme)
+ return;
+ if (lpfc_ncmd->ts_cmd_start < lpfc_ncmd->ts_last_cmd)
+ return;
+ if (lpfc_ncmd->ts_cmd_wqput < lpfc_ncmd->ts_cmd_start)
+ return;
+ if (lpfc_ncmd->ts_isr_cmpl < lpfc_ncmd->ts_cmd_wqput)
+ return;
+ if (lpfc_ncmd->ts_data_nvme < lpfc_ncmd->ts_isr_cmpl)
+ return;
+ /*
+ * Segment 1 - Time from Last FCP command cmpl is handed
+ * off to NVME Layer to start of next command.
+ * Segment 2 - Time from Driver receives a IO cmd start
+ * from NVME Layer to WQ put is done on IO cmd.
+ * Segment 3 - Time from Driver WQ put is done on IO cmd
+ * to MSI-X ISR for IO cmpl.
+ * Segment 4 - Time from MSI-X ISR for IO cmpl to when
+ * cmpl is handled off to the NVME Layer.
+ */
+ seg1 = lpfc_ncmd->ts_cmd_start - lpfc_ncmd->ts_last_cmd;
+ if (seg1 > 5000000) /* 5 ms - for sequential IOs */
+ return;
+
+ /* Calculate times relative to start of IO */
+ seg2 = (lpfc_ncmd->ts_cmd_wqput - lpfc_ncmd->ts_cmd_start);
+ seg3 = (lpfc_ncmd->ts_isr_cmpl -
+ lpfc_ncmd->ts_cmd_start) - seg2;
+ seg4 = (lpfc_ncmd->ts_data_nvme -
+ lpfc_ncmd->ts_cmd_start) - seg2 - seg3;
+ phba->ktime_data_samples++;
+ phba->ktime_seg1_total += seg1;
+ if (seg1 < phba->ktime_seg1_min)
+ phba->ktime_seg1_min = seg1;
+ else if (seg1 > phba->ktime_seg1_max)
+ phba->ktime_seg1_max = seg1;
+ phba->ktime_seg2_total += seg2;
+ if (seg2 < phba->ktime_seg2_min)
+ phba->ktime_seg2_min = seg2;
+ else if (seg2 > phba->ktime_seg2_max)
+ phba->ktime_seg2_max = seg2;
+ phba->ktime_seg3_total += seg3;
+ if (seg3 < phba->ktime_seg3_min)
+ phba->ktime_seg3_min = seg3;
+ else if (seg3 > phba->ktime_seg3_max)
+ phba->ktime_seg3_max = seg3;
+ phba->ktime_seg4_total += seg4;
+ if (seg4 < phba->ktime_seg4_min)
+ phba->ktime_seg4_min = seg4;
+ else if (seg4 > phba->ktime_seg4_max)
+ phba->ktime_seg4_max = seg4;
+
+ lpfc_ncmd->ts_last_cmd = 0;
+ lpfc_ncmd->ts_cmd_start = 0;
+ lpfc_ncmd->ts_cmd_wqput = 0;
+ lpfc_ncmd->ts_isr_cmpl = 0;
+ lpfc_ncmd->ts_data_nvme = 0;
+}
+#endif
+
+/**
+ * lpfc_nvme_io_cmd_wqe_cmpl - Complete an NVME-over-FCP IO
+ * @lpfc_pnvme: Pointer to the driver's nvme instance data
+ * @lpfc_nvme_lport: Pointer to the driver's local port data
+ * @lpfc_nvme_rport: Pointer to the rport getting the @lpfc_nvme_ereq
+ *
+ * Driver registers this routine as it io request handler. This
+ * routine issues an fcp WQE with data from the @lpfc_nvme_fcpreq
+ * data structure to the rport indicated in @lpfc_nvme_rport.
+ *
+ * Return value :
+ * 0 - Success
+ * TODO: What are the failure codes.
+ **/
+static void
+lpfc_nvme_io_cmd_wqe_cmpl(struct lpfc_hba *phba, struct lpfc_iocbq *pwqeIn,
+ struct lpfc_wcqe_complete *wcqe)
+{
+ struct lpfc_nvme_buf *lpfc_ncmd =
+ (struct lpfc_nvme_buf *)pwqeIn->context1;
+ struct lpfc_vport *vport = pwqeIn->vport;
+ struct nvmefc_fcp_req *nCmd;
+ struct nvme_fc_ersp_iu *ep;
+ struct nvme_fc_cmd_iu *cp;
+ struct lpfc_nvme_rport *rport;
+ struct lpfc_nodelist *ndlp;
+ unsigned long flags;
+ uint32_t code;
+ uint16_t cid, sqhd, data;
+ uint32_t *ptr;
+
+ /* Sanity check on return of outstanding command */
+ if (!lpfc_ncmd || !lpfc_ncmd->nvmeCmd || !lpfc_ncmd->nrport) {
+ lpfc_printf_vlog(vport, KERN_ERR, LOG_NODE | LOG_NVME_IOERR,
+ "6071 Completion pointers bad on wqe %p.\n",
+ wcqe);
+ return;
+ }
+ phba->fc4NvmeIoCmpls++;
+
+ nCmd = lpfc_ncmd->nvmeCmd;
+ rport = lpfc_ncmd->nrport;
+
+ lpfc_nvmeio_data(phba, "NVME FCP CMPL: xri x%x stat x%x parm x%x\n",
+ lpfc_ncmd->cur_iocbq.sli4_xritag,
+ bf_get(lpfc_wcqe_c_status, wcqe), wcqe->parameter);
+ /*
+ * Catch race where our node has transitioned, but the
+ * transport is still transitioning.
+ */
+ ndlp = rport->ndlp;
+ if (!ndlp || !NLP_CHK_NODE_ACT(ndlp)) {
+ lpfc_printf_vlog(vport, KERN_ERR, LOG_NODE | LOG_NVME_IOERR,
+ "6061 rport %p, ndlp %p, DID x%06x ndlp "
+ "not ready.\n",
+ rport, ndlp, rport->remoteport->port_id);
+
+ ndlp = lpfc_findnode_did(vport, rport->remoteport->port_id);
+ if (!ndlp) {
+ lpfc_printf_vlog(vport, KERN_ERR, LOG_NVME_IOERR,
+ "6062 Ignoring NVME cmpl. No ndlp\n");
+ goto out_err;
+ }
+ }
+
+ code = bf_get(lpfc_wcqe_c_code, wcqe);
+ if (code == CQE_CODE_NVME_ERSP) {
+ /* For this type of CQE, we need to rebuild the rsp */
+ ep = (struct nvme_fc_ersp_iu *)nCmd->rspaddr;
+
+ /*
+ * Get Command Id from cmd to plug into response. This
+ * code is not needed in the next NVME Transport drop.
+ */
+ cp = (struct nvme_fc_cmd_iu *)nCmd->cmdaddr;
+ cid = cp->sqe.common.command_id;
+
+ /*
+ * RSN is in CQE word 2
+ * SQHD is in CQE Word 3 bits 15:0
+ * Cmd Specific info is in CQE Word 1
+ * and in CQE Word 0 bits 15:0
+ */
+ sqhd = bf_get(lpfc_wcqe_c_sqhead, wcqe);
+
+ /* Now lets build the NVME ERSP IU */
+ ep->iu_len = cpu_to_be16(8);
+ ep->rsn = wcqe->parameter;
+ ep->xfrd_len = cpu_to_be32(nCmd->payload_length);
+ ep->rsvd12 = 0;
+ ptr = (uint32_t *)&ep->cqe.result.u64;
+ *ptr++ = wcqe->total_data_placed;
+ data = bf_get(lpfc_wcqe_c_ersp0, wcqe);
+ *ptr = (uint32_t)data;
+ ep->cqe.sq_head = sqhd;
+ ep->cqe.sq_id = nCmd->sqid;
+ ep->cqe.command_id = cid;
+ ep->cqe.status = 0;
+
+ lpfc_ncmd->status = IOSTAT_SUCCESS;
+ lpfc_ncmd->result = 0;
+ nCmd->rcv_rsplen = LPFC_NVME_ERSP_LEN;
+ nCmd->transferred_length = nCmd->payload_length;
+ } else {
+ lpfc_ncmd->status = (bf_get(lpfc_wcqe_c_status, wcqe) &
+ LPFC_IOCB_STATUS_MASK);
+ lpfc_ncmd->result = wcqe->parameter;
+
+ /* For NVME, the only failure path that results in an
+ * IO error is when the adapter rejects it. All other
+ * conditions are a success case and resolved by the
+ * transport.
+ * IOSTAT_FCP_RSP_ERROR means:
+ * 1. Length of data received doesn't match total
+ * transfer length in WQE
+ * 2. If the RSP payload does NOT match these cases:
+ * a. RSP length 12/24 bytes and all zeros
+ * b. NVME ERSP
+ */
+ switch (lpfc_ncmd->status) {
+ case IOSTAT_SUCCESS:
+ nCmd->transferred_length = wcqe->total_data_placed;
+ nCmd->rcv_rsplen = 0;
+ nCmd->status = 0;
+ break;
+ case IOSTAT_FCP_RSP_ERROR:
+ nCmd->transferred_length = wcqe->total_data_placed;
+ nCmd->rcv_rsplen = wcqe->parameter;
+ nCmd->status = 0;
+ /* Sanity check */
+ if (nCmd->rcv_rsplen == LPFC_NVME_ERSP_LEN)
+ break;
+ lpfc_printf_vlog(vport, KERN_ERR, LOG_NVME_IOERR,
+ "6081 NVME Completion Protocol Error: "
+ "status x%x result x%x placed x%x\n",
+ lpfc_ncmd->status, lpfc_ncmd->result,
+ wcqe->total_data_placed);
+ break;
+ default:
+out_err:
+ lpfc_printf_vlog(vport, KERN_ERR, LOG_NVME_IOERR,
+ "6072 NVME Completion Error: "
+ "status x%x result x%x placed x%x\n",
+ lpfc_ncmd->status, lpfc_ncmd->result,
+ wcqe->total_data_placed);
+ nCmd->transferred_length = 0;
+ nCmd->rcv_rsplen = 0;
+ nCmd->status = NVME_SC_FC_TRANSPORT_ERROR;
+ }
+ }
+
+ /* pick up SLI4 exhange busy condition */
+ if (bf_get(lpfc_wcqe_c_xb, wcqe))
+ lpfc_ncmd->flags |= LPFC_SBUF_XBUSY;
+ else
+ lpfc_ncmd->flags &= ~LPFC_SBUF_XBUSY;
+
+ if (ndlp && NLP_CHK_NODE_ACT(ndlp))
+ atomic_dec(&ndlp->cmd_pending);
+
+ /* Update stats and complete the IO. There is
+ * no need for dma unprep because the nvme_transport
+ * owns the dma address.
+ */
+#ifdef CONFIG_SCSI_LPFC_DEBUG_FS
+ if (phba->ktime_on) {
+ lpfc_ncmd->ts_isr_cmpl = pwqeIn->isr_timestamp;
+ lpfc_ncmd->ts_data_nvme = ktime_get_ns();
+ phba->ktime_last_cmd = lpfc_ncmd->ts_data_nvme;
+ lpfc_nvme_ktime(phba, lpfc_ncmd);
+ }
+ if (phba->cpucheck_on & LPFC_CHECK_NVME_IO) {
+ if (lpfc_ncmd->cpu != smp_processor_id())
+ lpfc_printf_vlog(vport, KERN_ERR, LOG_NVME_IOERR,
+ "6701 CPU Check cmpl: "
+ "cpu %d expect %d\n",
+ smp_processor_id(), lpfc_ncmd->cpu);
+ if (lpfc_ncmd->cpu < LPFC_CHECK_CPU_CNT)
+ phba->cpucheck_cmpl_io[lpfc_ncmd->cpu]++;
+ }
+#endif
+ nCmd->done(nCmd);
+
+ spin_lock_irqsave(&phba->hbalock, flags);
+ lpfc_ncmd->nrport = NULL;
+ spin_unlock_irqrestore(&phba->hbalock, flags);
+
+ lpfc_release_nvme_buf(phba, lpfc_ncmd);
+}
+
+
+/**
+ * lpfc_nvme_prep_io_cmd - Issue an NVME-over-FCP IO
+ * @lpfc_pnvme: Pointer to the driver's nvme instance data
+ * @lpfc_nvme_lport: Pointer to the driver's local port data
+ * @lpfc_nvme_rport: Pointer to the rport getting the @lpfc_nvme_ereq
+ * @lpfc_nvme_fcreq: IO request from nvme fc to driver.
+ * @hw_queue_handle: Driver-returned handle in lpfc_nvme_create_queue
+ *
+ * Driver registers this routine as it io request handler. This
+ * routine issues an fcp WQE with data from the @lpfc_nvme_fcpreq
+ * data structure to the rport indicated in @lpfc_nvme_rport.
+ *
+ * Return value :
+ * 0 - Success
+ * TODO: What are the failure codes.
+ **/
+static int
+lpfc_nvme_prep_io_cmd(struct lpfc_vport *vport,
+ struct lpfc_nvme_buf *lpfc_ncmd,
+ struct lpfc_nodelist *pnode)
+{
+ struct lpfc_hba *phba = vport->phba;
+ struct nvmefc_fcp_req *nCmd = lpfc_ncmd->nvmeCmd;
+ struct lpfc_iocbq *pwqeq = &(lpfc_ncmd->cur_iocbq);
+ union lpfc_wqe128 *wqe = (union lpfc_wqe128 *)&pwqeq->wqe;
+ uint32_t req_len;
+
+ if (!pnode || !NLP_CHK_NODE_ACT(pnode))
+ return -EINVAL;
+
+ /*
+ * There are three possibilities here - use scatter-gather segment, use
+ * the single mapping, or neither.
+ */
+ wqe->fcp_iwrite.initial_xfer_len = 0;
+ if (nCmd->sg_cnt) {
+ if (nCmd->io_dir == NVMEFC_FCP_WRITE) {
+ /* Word 5 */
+ if ((phba->cfg_nvme_enable_fb) &&
+ (pnode->nlp_flag & NLP_FIRSTBURST)) {
+ req_len = lpfc_ncmd->nvmeCmd->payload_length;
+ if (req_len < pnode->nvme_fb_size)
+ wqe->fcp_iwrite.initial_xfer_len =
+ req_len;
+ else
+ wqe->fcp_iwrite.initial_xfer_len =
+ pnode->nvme_fb_size;
+ }
+
+ /* Word 7 */
+ bf_set(wqe_cmnd, &wqe->generic.wqe_com,
+ CMD_FCP_IWRITE64_WQE);
+ bf_set(wqe_pu, &wqe->generic.wqe_com,
+ PARM_READ_CHECK);
+
+ /* Word 10 */
+ bf_set(wqe_qosd, &wqe->fcp_iwrite.wqe_com, 0);
+ bf_set(wqe_iod, &wqe->fcp_iwrite.wqe_com,
+ LPFC_WQE_IOD_WRITE);
+ bf_set(wqe_lenloc, &wqe->fcp_iwrite.wqe_com,
+ LPFC_WQE_LENLOC_WORD4);
+ if (phba->cfg_nvme_oas)
+ bf_set(wqe_oas, &wqe->fcp_iwrite.wqe_com, 1);
+
+ /* Word 11 */
+ bf_set(wqe_cmd_type, &wqe->generic.wqe_com,
+ NVME_WRITE_CMD);
+
+ /* Word 16 */
+ wqe->words[16] = LPFC_NVME_EMBED_WRITE;
+
+ phba->fc4NvmeOutputRequests++;
+ } else {
+ /* Word 7 */
+ bf_set(wqe_cmnd, &wqe->generic.wqe_com,
+ CMD_FCP_IREAD64_WQE);
+ bf_set(wqe_pu, &wqe->generic.wqe_com,
+ PARM_READ_CHECK);
+
+ /* Word 10 */
+ bf_set(wqe_qosd, &wqe->fcp_iread.wqe_com, 0);
+ bf_set(wqe_iod, &wqe->fcp_iread.wqe_com,
+ LPFC_WQE_IOD_READ);
+ bf_set(wqe_lenloc, &wqe->fcp_iread.wqe_com,
+ LPFC_WQE_LENLOC_WORD4);
+ if (phba->cfg_nvme_oas)
+ bf_set(wqe_oas, &wqe->fcp_iread.wqe_com, 1);
+
+ /* Word 11 */
+ bf_set(wqe_cmd_type, &wqe->generic.wqe_com,
+ NVME_READ_CMD);
+
+ /* Word 16 */
+ wqe->words[16] = LPFC_NVME_EMBED_READ;
+
+ phba->fc4NvmeInputRequests++;
+ }
+ } else {
+ /* Word 4 */
+ wqe->fcp_icmd.rsrvd4 = 0;
+
+ /* Word 7 */
+ bf_set(wqe_cmnd, &wqe->generic.wqe_com, CMD_FCP_ICMND64_WQE);
+ bf_set(wqe_pu, &wqe->generic.wqe_com, 0);
+
+ /* Word 10 */
+ bf_set(wqe_qosd, &wqe->fcp_icmd.wqe_com, 1);
+ bf_set(wqe_iod, &wqe->fcp_icmd.wqe_com, LPFC_WQE_IOD_WRITE);
+ bf_set(wqe_lenloc, &wqe->fcp_icmd.wqe_com,
+ LPFC_WQE_LENLOC_NONE);
+ if (phba->cfg_nvme_oas)
+ bf_set(wqe_oas, &wqe->fcp_icmd.wqe_com, 1);
+
+ /* Word 11 */
+ bf_set(wqe_cmd_type, &wqe->generic.wqe_com, NVME_READ_CMD);
+
+ /* Word 16 */
+ wqe->words[16] = LPFC_NVME_EMBED_CMD;
+
+ phba->fc4NvmeControlRequests++;
+ }
+ /*
+ * Finish initializing those WQE fields that are independent
+ * of the nvme_cmnd request_buffer
+ */
+
+ /* Word 6 */
+ bf_set(wqe_ctxt_tag, &wqe->generic.wqe_com,
+ phba->sli4_hba.rpi_ids[pnode->nlp_rpi]);
+ bf_set(wqe_xri_tag, &wqe->generic.wqe_com, pwqeq->sli4_xritag);
+
+ /* Word 7 */
+ /* Preserve Class data in the ndlp. */
+ bf_set(wqe_class, &wqe->generic.wqe_com,
+ (pnode->nlp_fcp_info & 0x0f));
+
+ /* Word 8 */
+ wqe->generic.wqe_com.abort_tag = pwqeq->iotag;
+
+ /* Word 9 */
+ bf_set(wqe_reqtag, &wqe->generic.wqe_com, pwqeq->iotag);
+
+ /* Word 11 */
+ bf_set(wqe_cqid, &wqe->generic.wqe_com, LPFC_WQE_CQ_ID_DEFAULT);
+
+ pwqeq->vport = vport;
+ return 0;
+}
+
+
+/**
+ * lpfc_nvme_prep_io_dma - Issue an NVME-over-FCP IO
+ * @lpfc_pnvme: Pointer to the driver's nvme instance data
+ * @lpfc_nvme_lport: Pointer to the driver's local port data
+ * @lpfc_nvme_rport: Pointer to the rport getting the @lpfc_nvme_ereq
+ * @lpfc_nvme_fcreq: IO request from nvme fc to driver.
+ * @hw_queue_handle: Driver-returned handle in lpfc_nvme_create_queue
+ *
+ * Driver registers this routine as it io request handler. This
+ * routine issues an fcp WQE with data from the @lpfc_nvme_fcpreq
+ * data structure to the rport indicated in @lpfc_nvme_rport.
+ *
+ * Return value :
+ * 0 - Success
+ * TODO: What are the failure codes.
+ **/
+static int
+lpfc_nvme_prep_io_dma(struct lpfc_vport *vport,
+ struct lpfc_nvme_buf *lpfc_ncmd)
+{
+ struct lpfc_hba *phba = vport->phba;
+ struct nvmefc_fcp_req *nCmd = lpfc_ncmd->nvmeCmd;
+ union lpfc_wqe128 *wqe = (union lpfc_wqe128 *)&lpfc_ncmd->cur_iocbq.wqe;
+ struct sli4_sge *sgl = lpfc_ncmd->nvme_sgl;
+ struct scatterlist *data_sg;
+ struct sli4_sge *first_data_sgl;
+ dma_addr_t physaddr;
+ uint32_t num_bde = 0;
+ uint32_t dma_len;
+ uint32_t dma_offset = 0;
+ int nseg, i;
+
+ /* Fix up the command and response DMA stuff. */
+ lpfc_nvme_adj_fcp_sgls(vport, lpfc_ncmd, nCmd);
+
+ /*
+ * There are three possibilities here - use scatter-gather segment, use
+ * the single mapping, or neither.
+ */
+ if (nCmd->sg_cnt) {
+ /*
+ * Jump over the cmd and rsp SGEs. The fix routine
+ * has already adjusted for this.
+ */
+ sgl += 2;
+
+ first_data_sgl = sgl;
+ lpfc_ncmd->seg_cnt = nCmd->sg_cnt;
+ if (lpfc_ncmd->seg_cnt > phba->cfg_sg_seg_cnt) {
+ lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR,
+ "6058 Too many sg segments from "
+ "NVME Transport. Max %d, "
+ "nvmeIO sg_cnt %d\n",
+ phba->cfg_sg_seg_cnt,
+ lpfc_ncmd->seg_cnt);
+ lpfc_ncmd->seg_cnt = 0;
+ return 1;
+ }
+
+ /*
+ * The driver established a maximum scatter-gather segment count
+ * during probe that limits the number of sg elements in any
+ * single nvme command. Just run through the seg_cnt and format
+ * the sge's.
+ */
+ nseg = nCmd->sg_cnt;
+ data_sg = nCmd->first_sgl;
+ for (i = 0; i < nseg; i++) {
+ if (data_sg == NULL) {
+ lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR,
+ "6059 dptr err %d, nseg %d\n",
+ i, nseg);
+ lpfc_ncmd->seg_cnt = 0;
+ return 1;
+ }
+ physaddr = data_sg->dma_address;
+ dma_len = data_sg->length;
+ sgl->addr_lo = cpu_to_le32(putPaddrLow(physaddr));
+ sgl->addr_hi = cpu_to_le32(putPaddrHigh(physaddr));
+ sgl->word2 = le32_to_cpu(sgl->word2);
+ if ((num_bde + 1) == nseg)
+ bf_set(lpfc_sli4_sge_last, sgl, 1);
+ else
+ bf_set(lpfc_sli4_sge_last, sgl, 0);
+ bf_set(lpfc_sli4_sge_offset, sgl, dma_offset);
+ bf_set(lpfc_sli4_sge_type, sgl, LPFC_SGE_TYPE_DATA);
+ sgl->word2 = cpu_to_le32(sgl->word2);
+ sgl->sge_len = cpu_to_le32(dma_len);
+
+ dma_offset += dma_len;
+ data_sg = sg_next(data_sg);
+ sgl++;
+ }
+ } else {
+ /* For this clause to be valid, the payload_length
+ * and sg_cnt must zero.
+ */
+ if (nCmd->payload_length != 0) {
+ lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR,
+ "6063 NVME DMA Prep Err: sg_cnt %d "
+ "payload_length x%x\n",
+ nCmd->sg_cnt, nCmd->payload_length);
+ return 1;
+ }
+ }
+
+ /*
+ * Due to difference in data length between DIF/non-DIF paths,
+ * we need to set word 4 of WQE here
+ */
+ wqe->fcp_iread.total_xfer_len = nCmd->payload_length;
+ return 0;
+}
+
+/**
+ * lpfc_nvme_fcp_io_submit - Issue an NVME-over-FCP IO
+ * @lpfc_pnvme: Pointer to the driver's nvme instance data
+ * @lpfc_nvme_lport: Pointer to the driver's local port data
+ * @lpfc_nvme_rport: Pointer to the rport getting the @lpfc_nvme_ereq
+ * @lpfc_nvme_fcreq: IO request from nvme fc to driver.
+ * @hw_queue_handle: Driver-returned handle in lpfc_nvme_create_queue
+ *
+ * Driver registers this routine as it io request handler. This
+ * routine issues an fcp WQE with data from the @lpfc_nvme_fcpreq
+ * data structure to the rport
+ indicated in @lpfc_nvme_rport.
+ *
+ * Return value :
+ * 0 - Success
+ * TODO: What are the failure codes.
+ **/
+static int
+lpfc_nvme_fcp_io_submit(struct nvme_fc_local_port *pnvme_lport,
+ struct nvme_fc_remote_port *pnvme_rport,
+ void *hw_queue_handle,
+ struct nvmefc_fcp_req *pnvme_fcreq)
+{
+ int ret = 0;
+ struct lpfc_nvme_lport *lport;
+ struct lpfc_vport *vport;
+ struct lpfc_hba *phba;
+ struct lpfc_nodelist *ndlp;
+ struct lpfc_nvme_buf *lpfc_ncmd;
+ struct lpfc_nvme_rport *rport;
+ struct lpfc_nvme_qhandle *lpfc_queue_info;
+#ifdef CONFIG_SCSI_LPFC_DEBUG_FS
+ uint64_t start = 0;
+#endif
+
+ lport = (struct lpfc_nvme_lport *)pnvme_lport->private;
+ vport = lport->vport;
+ phba = vport->phba;
+
+#ifdef CONFIG_SCSI_LPFC_DEBUG_FS
+ if (phba->ktime_on)
+ start = ktime_get_ns();
+#endif
+ rport = (struct lpfc_nvme_rport *)pnvme_rport->private;
+ lpfc_queue_info = (struct lpfc_nvme_qhandle *)hw_queue_handle;
+
+ /*
+ * Catch race where our node has transitioned, but the
+ * transport is still transitioning.
+ */
+ ndlp = rport->ndlp;
+ if (!ndlp || !NLP_CHK_NODE_ACT(ndlp)) {
+ lpfc_printf_vlog(vport, KERN_ERR, LOG_NODE | LOG_NVME_IOERR,
+ "6053 rport %p, ndlp %p, DID x%06x "
+ "ndlp not ready.\n",
+ rport, ndlp, pnvme_rport->port_id);
+
+ ndlp = lpfc_findnode_did(vport, pnvme_rport->port_id);
+ if (!ndlp) {
+ lpfc_printf_vlog(vport, KERN_ERR, LOG_NVME_IOERR,
+ "6066 Missing node for DID %x\n",
+ pnvme_rport->port_id);
+ ret = -ENODEV;
+ goto out_fail;
+ }
+ }
+
+ /* The remote node has to be a mapped target or it's an error. */
+ if ((ndlp->nlp_type & NLP_NVME_TARGET) &&
+ (ndlp->nlp_state != NLP_STE_MAPPED_NODE)) {
+ lpfc_printf_vlog(vport, KERN_ERR, LOG_NODE | LOG_NVME_IOERR,
+ "6036 rport %p, DID x%06x not ready for "
+ "IO. State x%x, Type x%x\n",
+ rport, pnvme_rport->port_id,
+ ndlp->nlp_state, ndlp->nlp_type);
+ ret = -ENODEV;
+ goto out_fail;
+
+ }
+
+ /* The node is shared with FCP IO, make sure the IO pending count does
+ * not exceed the programmed depth.
+ */
+ if (atomic_read(&ndlp->cmd_pending) >= ndlp->cmd_qdepth) {
+ ret = -EAGAIN;
+ goto out_fail;
+ }
+
+ lpfc_ncmd = lpfc_get_nvme_buf(phba, ndlp);
+ if (lpfc_ncmd == NULL) {
+ lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_IOERR,
+ "6065 driver's buffer pool is empty, "
+ "IO failed\n");
+ ret = -ENOMEM;
+ goto out_fail;
+ }
+#ifdef CONFIG_SCSI_LPFC_DEBUG_FS
+ if (phba->ktime_on) {
+ lpfc_ncmd->ts_cmd_start = start;
+ lpfc_ncmd->ts_last_cmd = phba->ktime_last_cmd;
+ }
+#endif
+
+ /*
+ * Store the data needed by the driver to issue, abort, and complete
+ * an IO.
+ * Do not let the IO hang out forever. There is no midlayer issuing
+ * an abort so inform the FW of the maximum IO pending time.
+ */
+ pnvme_fcreq->private = (void *)lpfc_ncmd;
+ lpfc_ncmd->nvmeCmd = pnvme_fcreq;
+ lpfc_ncmd->nrport = rport;
+ lpfc_ncmd->start_time = jiffies;
+
+ lpfc_nvme_prep_io_cmd(vport, lpfc_ncmd, ndlp);
+ ret = lpfc_nvme_prep_io_dma(vport, lpfc_ncmd);
+ if (ret) {
+ ret = -ENOMEM;
+ goto out_free_nvme_buf;
+ }
+
+ atomic_inc(&ndlp->cmd_pending);
+
+ /*
+ * Issue the IO on the WQ indicated by index in the hw_queue_handle.
+ * This identfier was create in our hardware queue create callback
+ * routine. The driver now is dependent on the IO queue steering from
+ * the transport. We are trusting the upper NVME layers know which
+ * index to use and that they have affinitized a CPU to this hardware
+ * queue. A hardware queue maps to a driver MSI-X vector/EQ/CQ/WQ.
+ */
+ lpfc_ncmd->cur_iocbq.hba_wqidx = lpfc_queue_info->index;
+
+ lpfc_nvmeio_data(phba, "NVME FCP XMIT: xri x%x idx %d to %06x\n",
+ lpfc_ncmd->cur_iocbq.sli4_xritag,
+ lpfc_queue_info->index, ndlp->nlp_DID);
+
+ ret = lpfc_sli4_issue_wqe(phba, LPFC_FCP_RING, &lpfc_ncmd->cur_iocbq);
+ if (ret) {
+ atomic_dec(&ndlp->cmd_pending);
+ lpfc_printf_vlog(vport, KERN_ERR, LOG_NVME_IOERR,
+ "6113 FCP could not issue WQE err %x "
+ "sid: x%x did: x%x oxid: x%x\n",
+ ret, vport->fc_myDID, ndlp->nlp_DID,
+ lpfc_ncmd->cur_iocbq.sli4_xritag);
+ ret = -EINVAL;
+ goto out_free_nvme_buf;
+ }
+
+#ifdef CONFIG_SCSI_LPFC_DEBUG_FS
+ if (phba->ktime_on)
+ lpfc_ncmd->ts_cmd_wqput = ktime_get_ns();
+
+ if (phba->cpucheck_on & LPFC_CHECK_NVME_IO) {
+ lpfc_ncmd->cpu = smp_processor_id();
+ if (lpfc_ncmd->cpu != lpfc_queue_info->index) {
+ /* Check for admin queue */
+ if (lpfc_queue_info->qidx) {
+ lpfc_printf_vlog(vport,
+ KERN_ERR, LOG_NVME_IOERR,
+ "6702 CPU Check cmd: "
+ "cpu %d wq %d\n",
+ lpfc_ncmd->cpu,
+ lpfc_queue_info->index);
+ }
+ lpfc_ncmd->cpu = lpfc_queue_info->index;
+ }
+ if (lpfc_ncmd->cpu < LPFC_CHECK_CPU_CNT)
+ phba->cpucheck_xmt_io[lpfc_ncmd->cpu]++;
+ }
+#endif
+ return 0;
+
+ out_free_nvme_buf:
+ lpfc_release_nvme_buf(phba, lpfc_ncmd);
+ out_fail:
+ return ret;
+}
+
+/**
+ * lpfc_nvme_abort_fcreq_cmpl - Complete an NVME FCP abort request.
+ * @phba: Pointer to HBA context object
+ * @cmdiocb: Pointer to command iocb object.
+ * @rspiocb: Pointer to response iocb object.
+ *
+ * This is the callback function for any NVME FCP IO that was aborted.
+ *
+ * Return value:
+ * None
+ **/
+void
+lpfc_nvme_abort_fcreq_cmpl(struct lpfc_hba *phba, struct lpfc_iocbq *cmdiocb,
+ struct lpfc_wcqe_complete *abts_cmpl)
+{
+ lpfc_printf_log(phba, KERN_ERR, LOG_NVME,
+ "6145 ABORT_XRI_CN completing on rpi x%x "
+ "original iotag x%x, abort cmd iotag x%x "
+ "req_tag x%x, status x%x, hwstatus x%x\n",
+ cmdiocb->iocb.un.acxri.abortContextTag,
+ cmdiocb->iocb.un.acxri.abortIoTag,
+ cmdiocb->iotag,
+ bf_get(lpfc_wcqe_c_request_tag, abts_cmpl),
+ bf_get(lpfc_wcqe_c_status, abts_cmpl),
+ bf_get(lpfc_wcqe_c_hw_status, abts_cmpl));
+ lpfc_sli_release_iocbq(phba, cmdiocb);
+}
+
+/**
+ * lpfc_nvme_fcp_abort - Issue an NVME-over-FCP ABTS
+ * @lpfc_pnvme: Pointer to the driver's nvme instance data
+ * @lpfc_nvme_lport: Pointer to the driver's local port data
+ * @lpfc_nvme_rport: Pointer to the rport getting the @lpfc_nvme_ereq
+ * @lpfc_nvme_fcreq: IO request from nvme fc to driver.
+ * @hw_queue_handle: Driver-returned handle in lpfc_nvme_create_queue
+ *
+ * Driver registers this routine as its nvme request io abort handler. This
+ * routine issues an fcp Abort WQE with data from the @lpfc_nvme_fcpreq
+ * data structure to the rport indicated in @lpfc_nvme_rport. This routine
+ * is executed asynchronously - one the target is validated as "MAPPED" and
+ * ready for IO, the driver issues the abort request and returns.
+ *
+ * Return value:
+ * None
+ **/
+static void
+lpfc_nvme_fcp_abort(struct nvme_fc_local_port *pnvme_lport,
+ struct nvme_fc_remote_port *pnvme_rport,
+ void *hw_queue_handle,
+ struct nvmefc_fcp_req *pnvme_fcreq)
+{
+ struct lpfc_nvme_lport *lport;
+ struct lpfc_vport *vport;
+ struct lpfc_hba *phba;
+ struct lpfc_nodelist *ndlp;
+ struct lpfc_nvme_rport *rport;
+ struct lpfc_nvme_buf *lpfc_nbuf;
+ struct lpfc_iocbq *abts_buf;
+ struct lpfc_iocbq *nvmereq_wqe;
+ union lpfc_wqe *abts_wqe;
+ unsigned long flags;
+ int ret_val;
+
+ lport = (struct lpfc_nvme_lport *)pnvme_lport->private;
+ rport = (struct lpfc_nvme_rport *)pnvme_rport->private;
+ vport = lport->vport;
+ phba = vport->phba;
+
+ /* Announce entry to new IO submit field. */
+ lpfc_printf_vlog(vport, KERN_ERR, LOG_NVME_ABTS,
+ "6002 Abort Request to rport DID x%06x "
+ "for nvme_fc_req %p\n",
+ pnvme_rport->port_id,
+ pnvme_fcreq);
+
+ /*
+ * Catch race where our node has transitioned, but the
+ * transport is still transitioning.
+ */
+ ndlp = rport->ndlp;
+ if (!ndlp || !NLP_CHK_NODE_ACT(ndlp)) {
+ lpfc_printf_vlog(vport, KERN_ERR, LOG_NODE | LOG_NVME_ABTS,
+ "6054 rport %p, ndlp %p, DID x%06x ndlp "
+ " not ready.\n",
+ rport, ndlp, pnvme_rport->port_id);
+
+ ndlp = lpfc_findnode_did(vport, pnvme_rport->port_id);
+ if (!ndlp) {
+ lpfc_printf_vlog(vport, KERN_ERR, LOG_NVME_ABTS,
+ "6055 Could not find node for "
+ "DID %x\n",
+ pnvme_rport->port_id);
+ return;
+ }
+ }
+
+ /* The remote node has to be ready to send an abort. */
+ if ((ndlp->nlp_state != NLP_STE_MAPPED_NODE) &&
+ !(ndlp->nlp_type & NLP_NVME_TARGET)) {
+ lpfc_printf_vlog(vport, KERN_ERR, LOG_NODE | LOG_NVME_ABTS,
+ "6048 rport %p, DID x%06x not ready for "
+ "IO. State x%x, Type x%x\n",
+ rport, pnvme_rport->port_id,
+ ndlp->nlp_state, ndlp->nlp_type);
+ return;
+ }
+
+ /* If the hba is getting reset, this flag is set. It is
+ * cleared when the reset is complete and rings reestablished.
+ */
+ spin_lock_irqsave(&phba->hbalock, flags);
+ /* driver queued commands are in process of being flushed */
+ if (phba->hba_flag & HBA_NVME_IOQ_FLUSH) {
+ spin_unlock_irqrestore(&phba->hbalock, flags);
+ lpfc_printf_vlog(vport, KERN_ERR, LOG_NVME,
+ "6139 Driver in reset cleanup - flushing "
+ "NVME Req now. hba_flag x%x\n",
+ phba->hba_flag);
+ return;
+ }
+
+ lpfc_nbuf = (struct lpfc_nvme_buf *)pnvme_fcreq->private;
+ if (!lpfc_nbuf) {
+ spin_unlock_irqrestore(&phba->hbalock, flags);
+ lpfc_printf_vlog(vport, KERN_ERR, LOG_NVME,
+ "6140 NVME IO req has no matching lpfc nvme "
+ "io buffer. Skipping abort req.\n");
+ return;
+ } else if (!lpfc_nbuf->nvmeCmd) {
+ spin_unlock_irqrestore(&phba->hbalock, flags);
+ lpfc_printf_vlog(vport, KERN_ERR, LOG_NVME,
+ "6141 lpfc NVME IO req has no nvme_fcreq "
+ "io buffer. Skipping abort req.\n");
+ return;
+ }
+
+ /*
+ * The lpfc_nbuf and the mapped nvme_fcreq in the driver's
+ * state must match the nvme_fcreq passed by the nvme
+ * transport. If they don't match, it is likely the driver
+ * has already completed the NVME IO and the nvme transport
+ * has not seen it yet.
+ */
+ if (lpfc_nbuf->nvmeCmd != pnvme_fcreq) {
+ spin_unlock_irqrestore(&phba->hbalock, flags);
+ lpfc_printf_vlog(vport, KERN_ERR, LOG_NVME,
+ "6143 NVME req mismatch: "
+ "lpfc_nbuf %p nvmeCmd %p, "
+ "pnvme_fcreq %p. Skipping Abort\n",
+ lpfc_nbuf, lpfc_nbuf->nvmeCmd,
+ pnvme_fcreq);
+ return;
+ }
+
+ /* Don't abort IOs no longer on the pending queue. */
+ nvmereq_wqe = &lpfc_nbuf->cur_iocbq;
+ if (!(nvmereq_wqe->iocb_flag & LPFC_IO_ON_TXCMPLQ)) {
+ spin_unlock_irqrestore(&phba->hbalock, flags);
+ lpfc_printf_vlog(vport, KERN_ERR, LOG_NVME,
+ "6142 NVME IO req %p not queued - skipping "
+ "abort req\n",
+ pnvme_fcreq);
+ return;
+ }
+
+ lpfc_nvmeio_data(phba, "NVME FCP ABORT: xri x%x idx %d to %06x\n",
+ nvmereq_wqe->sli4_xritag,
+ nvmereq_wqe->hba_wqidx, ndlp->nlp_DID);
+
+ /* Outstanding abort is in progress */
+ if (nvmereq_wqe->iocb_flag & LPFC_DRIVER_ABORTED) {
+ spin_unlock_irqrestore(&phba->hbalock, flags);
+ lpfc_printf_vlog(vport, KERN_ERR, LOG_NVME,
+ "6144 Outstanding NVME I/O Abort Request "
+ "still pending on nvme_fcreq %p, "
+ "lpfc_ncmd %p\n",
+ pnvme_fcreq, lpfc_nbuf);
+ return;
+ }
+
+ abts_buf = __lpfc_sli_get_iocbq(phba);
+ if (!abts_buf) {
+ spin_unlock_irqrestore(&phba->hbalock, flags);
+ lpfc_printf_vlog(vport, KERN_ERR, LOG_NVME,
+ "6136 No available abort wqes. Skipping "
+ "Abts req for nvme_fcreq %p.\n",
+ pnvme_fcreq);
+ return;
+ }
+
+ /* Ready - mark outstanding as aborted by driver. */
+ nvmereq_wqe->iocb_flag |= LPFC_DRIVER_ABORTED;
+
+ /* Complete prepping the abort wqe and issue to the FW. */
+ abts_wqe = &abts_buf->wqe;
+
+ /* WQEs are reused. Clear stale data and set key fields to
+ * zero like ia, iaab, iaar, xri_tag, and ctxt_tag.
+ */
+ memset(abts_wqe, 0, sizeof(union lpfc_wqe));
+ bf_set(abort_cmd_criteria, &abts_wqe->abort_cmd, T_XRI_TAG);
+
+ /* word 7 */
+ bf_set(wqe_ct, &abts_wqe->abort_cmd.wqe_com, 0);
+ bf_set(wqe_cmnd, &abts_wqe->abort_cmd.wqe_com, CMD_ABORT_XRI_CX);
+ bf_set(wqe_class, &abts_wqe->abort_cmd.wqe_com,
+ nvmereq_wqe->iocb.ulpClass);
+
+ /* word 8 - tell the FW to abort the IO associated with this
+ * outstanding exchange ID.
+ */
+ abts_wqe->abort_cmd.wqe_com.abort_tag = nvmereq_wqe->sli4_xritag;
+
+ /* word 9 - this is the iotag for the abts_wqe completion. */
+ bf_set(wqe_reqtag, &abts_wqe->abort_cmd.wqe_com,
+ abts_buf->iotag);
+
+ /* word 10 */
+ bf_set(wqe_wqid, &abts_wqe->abort_cmd.wqe_com, nvmereq_wqe->hba_wqidx);
+ bf_set(wqe_qosd, &abts_wqe->abort_cmd.wqe_com, 1);
+ bf_set(wqe_lenloc, &abts_wqe->abort_cmd.wqe_com, LPFC_WQE_LENLOC_NONE);
+
+ /* word 11 */
+ bf_set(wqe_cmd_type, &abts_wqe->abort_cmd.wqe_com, OTHER_COMMAND);
+ bf_set(wqe_wqec, &abts_wqe->abort_cmd.wqe_com, 1);
+ bf_set(wqe_cqid, &abts_wqe->abort_cmd.wqe_com, LPFC_WQE_CQ_ID_DEFAULT);
+
+ /* ABTS WQE must go to the same WQ as the WQE to be aborted */
+ abts_buf->iocb_flag |= LPFC_IO_NVME;
+ abts_buf->hba_wqidx = nvmereq_wqe->hba_wqidx;
+ abts_buf->vport = vport;
+ abts_buf->wqe_cmpl = lpfc_nvme_abort_fcreq_cmpl;
+ ret_val = lpfc_sli4_issue_wqe(phba, LPFC_FCP_RING, abts_buf);
+ spin_unlock_irqrestore(&phba->hbalock, flags);
+ if (ret_val == IOCB_ERROR) {
+ lpfc_printf_vlog(vport, KERN_ERR, LOG_NVME,
+ "6137 Failed abts issue_wqe with status x%x "
+ "for nvme_fcreq %p.\n",
+ ret_val, pnvme_fcreq);
+ lpfc_sli_release_iocbq(phba, abts_buf);
+ return;
+ }
+
+ lpfc_printf_vlog(vport, KERN_ERR, LOG_NVME,
+ "6138 Transport Abort NVME Request Issued for\n"
+ "ox_id x%x on reqtag x%x\n",
+ nvmereq_wqe->sli4_xritag,
+ abts_buf->iotag);
+}
+
+/* Declare and initialization an instance of the FC NVME template. */
+static struct nvme_fc_port_template lpfc_nvme_template = {
+ /* initiator-based functions */
+ .localport_delete = lpfc_nvme_localport_delete,
+ .remoteport_delete = lpfc_nvme_remoteport_delete,
+ .create_queue = lpfc_nvme_create_queue,
+ .delete_queue = lpfc_nvme_delete_queue,
+ .ls_req = lpfc_nvme_ls_req,
+ .fcp_io = lpfc_nvme_fcp_io_submit,
+ .ls_abort = lpfc_nvme_ls_abort,
+ .fcp_abort = lpfc_nvme_fcp_abort,
+
+ .max_hw_queues = 1,
+ .max_sgl_segments = LPFC_NVME_DEFAULT_SEGS,
+ .max_dif_sgl_segments = LPFC_NVME_DEFAULT_SEGS,
+ .dma_boundary = 0xFFFFFFFF,
+
+ /* Sizes of additional private data for data structures.
+ * No use for the last two sizes at this time.
+ */
+ .local_priv_sz = sizeof(struct lpfc_nvme_lport),
+ .remote_priv_sz = sizeof(struct lpfc_nvme_rport),
+ .lsrqst_priv_sz = 0,
+ .fcprqst_priv_sz = 0,
+};
+
+/**
+ * lpfc_sli4_post_nvme_sgl_block - post a block of nvme sgl list to firmware
+ * @phba: pointer to lpfc hba data structure.
+ * @nblist: pointer to nvme buffer list.
+ * @count: number of scsi buffers on the list.
+ *
+ * This routine is invoked to post a block of @count scsi sgl pages from a
+ * SCSI buffer list @nblist to the HBA using non-embedded mailbox command.
+ * No Lock is held.
+ *
+ **/
+static int
+lpfc_sli4_post_nvme_sgl_block(struct lpfc_hba *phba,
+ struct list_head *nblist,
+ int count)
+{
+ struct lpfc_nvme_buf *lpfc_ncmd;
+ struct lpfc_mbx_post_uembed_sgl_page1 *sgl;
+ struct sgl_page_pairs *sgl_pg_pairs;
+ void *viraddr;
+ LPFC_MBOXQ_t *mbox;
+ uint32_t reqlen, alloclen, pg_pairs;
+ uint32_t mbox_tmo;
+ uint16_t xritag_start = 0;
+ int rc = 0;
+ uint32_t shdr_status, shdr_add_status;
+ dma_addr_t pdma_phys_bpl1;
+ union lpfc_sli4_cfg_shdr *shdr;
+
+ /* Calculate the requested length of the dma memory */
+ reqlen = count * sizeof(struct sgl_page_pairs) +
+ sizeof(union lpfc_sli4_cfg_shdr) + sizeof(uint32_t);
+ if (reqlen > SLI4_PAGE_SIZE) {
+ lpfc_printf_log(phba, KERN_WARNING, LOG_INIT,
+ "6118 Block sgl registration required DMA "
+ "size (%d) great than a page\n", reqlen);
+ return -ENOMEM;
+ }
+ mbox = mempool_alloc(phba->mbox_mem_pool, GFP_KERNEL);
+ if (!mbox) {
+ lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+ "6119 Failed to allocate mbox cmd memory\n");
+ return -ENOMEM;
+ }
+
+ /* Allocate DMA memory and set up the non-embedded mailbox command */
+ alloclen = lpfc_sli4_config(phba, mbox, LPFC_MBOX_SUBSYSTEM_FCOE,
+ LPFC_MBOX_OPCODE_FCOE_POST_SGL_PAGES, reqlen,
+ LPFC_SLI4_MBX_NEMBED);
+
+ if (alloclen < reqlen) {
+ lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+ "6120 Allocated DMA memory size (%d) is "
+ "less than the requested DMA memory "
+ "size (%d)\n", alloclen, reqlen);
+ lpfc_sli4_mbox_cmd_free(phba, mbox);
+ return -ENOMEM;
+ }
+
+ /* Get the first SGE entry from the non-embedded DMA memory */
+ viraddr = mbox->sge_array->addr[0];
+
+ /* Set up the SGL pages in the non-embedded DMA pages */
+ sgl = (struct lpfc_mbx_post_uembed_sgl_page1 *)viraddr;
+ sgl_pg_pairs = &sgl->sgl_pg_pairs;
+
+ pg_pairs = 0;
+ list_for_each_entry(lpfc_ncmd, nblist, list) {
+ /* Set up the sge entry */
+ sgl_pg_pairs->sgl_pg0_addr_lo =
+ cpu_to_le32(putPaddrLow(lpfc_ncmd->dma_phys_sgl));
+ sgl_pg_pairs->sgl_pg0_addr_hi =
+ cpu_to_le32(putPaddrHigh(lpfc_ncmd->dma_phys_sgl));
+ if (phba->cfg_sg_dma_buf_size > SGL_PAGE_SIZE)
+ pdma_phys_bpl1 = lpfc_ncmd->dma_phys_sgl +
+ SGL_PAGE_SIZE;
+ else
+ pdma_phys_bpl1 = 0;
+ sgl_pg_pairs->sgl_pg1_addr_lo =
+ cpu_to_le32(putPaddrLow(pdma_phys_bpl1));
+ sgl_pg_pairs->sgl_pg1_addr_hi =
+ cpu_to_le32(putPaddrHigh(pdma_phys_bpl1));
+ /* Keep the first xritag on the list */
+ if (pg_pairs == 0)
+ xritag_start = lpfc_ncmd->cur_iocbq.sli4_xritag;
+ sgl_pg_pairs++;
+ pg_pairs++;
+ }
+ bf_set(lpfc_post_sgl_pages_xri, sgl, xritag_start);
+ bf_set(lpfc_post_sgl_pages_xricnt, sgl, pg_pairs);
+ /* Perform endian conversion if necessary */
+ sgl->word0 = cpu_to_le32(sgl->word0);
+
+ if (!phba->sli4_hba.intr_enable)
+ rc = lpfc_sli_issue_mbox(phba, mbox, MBX_POLL);
+ else {
+ mbox_tmo = lpfc_mbox_tmo_val(phba, mbox);
+ rc = lpfc_sli_issue_mbox_wait(phba, mbox, mbox_tmo);
+ }
+ shdr = (union lpfc_sli4_cfg_shdr *)&sgl->cfg_shdr;
+ shdr_status = bf_get(lpfc_mbox_hdr_status, &shdr->response);
+ shdr_add_status = bf_get(lpfc_mbox_hdr_add_status, &shdr->response);
+ if (rc != MBX_TIMEOUT)
+ lpfc_sli4_mbox_cmd_free(phba, mbox);
+ if (shdr_status || shdr_add_status || rc) {
+ lpfc_printf_log(phba, KERN_ERR, LOG_SLI,
+ "6125 POST_SGL_BLOCK mailbox command failed "
+ "status x%x add_status x%x mbx status x%x\n",
+ shdr_status, shdr_add_status, rc);
+ rc = -ENXIO;
+ }
+ return rc;
+}
+
+/**
+ * lpfc_post_nvme_sgl_list - Post blocks of nvme buffer sgls from a list
+ * @phba: pointer to lpfc hba data structure.
+ * @post_nblist: pointer to the nvme buffer list.
+ *
+ * This routine walks a list of nvme buffers that was passed in. It attempts
+ * to construct blocks of nvme buffer sgls which contains contiguous xris and
+ * uses the non-embedded SGL block post mailbox commands to post to the port.
+ * For single NVME buffer sgl with non-contiguous xri, if any, it shall use
+ * embedded SGL post mailbox command for posting. The @post_nblist passed in
+ * must be local list, thus no lock is needed when manipulate the list.
+ *
+ * Returns: 0 = failure, non-zero number of successfully posted buffers.
+ **/
+static int
+lpfc_post_nvme_sgl_list(struct lpfc_hba *phba,
+ struct list_head *post_nblist, int sb_count)
+{
+ struct lpfc_nvme_buf *lpfc_ncmd, *lpfc_ncmd_next;
+ int status, sgl_size;
+ int post_cnt = 0, block_cnt = 0, num_posting = 0, num_posted = 0;
+ dma_addr_t pdma_phys_sgl1;
+ int last_xritag = NO_XRI;
+ int cur_xritag;
+ LIST_HEAD(prep_nblist);
+ LIST_HEAD(blck_nblist);
+ LIST_HEAD(nvme_nblist);
+
+ /* sanity check */
+ if (sb_count <= 0)
+ return -EINVAL;
+
+ sgl_size = phba->cfg_sg_dma_buf_size;
+
+ list_for_each_entry_safe(lpfc_ncmd, lpfc_ncmd_next, post_nblist, list) {
+ list_del_init(&lpfc_ncmd->list);
+ block_cnt++;
+ if ((last_xritag != NO_XRI) &&
+ (lpfc_ncmd->cur_iocbq.sli4_xritag != last_xritag + 1)) {
+ /* a hole in xri block, form a sgl posting block */
+ list_splice_init(&prep_nblist, &blck_nblist);
+ post_cnt = block_cnt - 1;
+ /* prepare list for next posting block */
+ list_add_tail(&lpfc_ncmd->list, &prep_nblist);
+ block_cnt = 1;
+ } else {
+ /* prepare list for next posting block */
+ list_add_tail(&lpfc_ncmd->list, &prep_nblist);
+ /* enough sgls for non-embed sgl mbox command */
+ if (block_cnt == LPFC_NEMBED_MBOX_SGL_CNT) {
+ list_splice_init(&prep_nblist, &blck_nblist);
+ post_cnt = block_cnt;
+ block_cnt = 0;
+ }
+ }
+ num_posting++;
+ last_xritag = lpfc_ncmd->cur_iocbq.sli4_xritag;
+
+ /* end of repost sgl list condition for NVME buffers */
+ if (num_posting == sb_count) {
+ if (post_cnt == 0) {
+ /* last sgl posting block */
+ list_splice_init(&prep_nblist, &blck_nblist);
+ post_cnt = block_cnt;
+ } else if (block_cnt == 1) {
+ /* last single sgl with non-contiguous xri */
+ if (sgl_size > SGL_PAGE_SIZE)
+ pdma_phys_sgl1 =
+ lpfc_ncmd->dma_phys_sgl +
+ SGL_PAGE_SIZE;
+ else
+ pdma_phys_sgl1 = 0;
+ cur_xritag = lpfc_ncmd->cur_iocbq.sli4_xritag;
+ status = lpfc_sli4_post_sgl(phba,
+ lpfc_ncmd->dma_phys_sgl,
+ pdma_phys_sgl1, cur_xritag);
+ if (status) {
+ /* failure, put on abort nvme list */
+ lpfc_ncmd->exch_busy = 1;
+ } else {
+ /* success, put on NVME buffer list */
+ lpfc_ncmd->exch_busy = 0;
+ lpfc_ncmd->status = IOSTAT_SUCCESS;
+ num_posted++;
+ }
+ /* success, put on NVME buffer sgl list */
+ list_add_tail(&lpfc_ncmd->list, &nvme_nblist);
+ }
+ }
+
+ /* continue until a nembed page worth of sgls */
+ if (post_cnt == 0)
+ continue;
+
+ /* post block of NVME buffer list sgls */
+ status = lpfc_sli4_post_nvme_sgl_block(phba, &blck_nblist,
+ post_cnt);
+
+ /* don't reset xirtag due to hole in xri block */
+ if (block_cnt == 0)
+ last_xritag = NO_XRI;
+
+ /* reset NVME buffer post count for next round of posting */
+ post_cnt = 0;
+
+ /* put posted NVME buffer-sgl posted on NVME buffer sgl list */
+ while (!list_empty(&blck_nblist)) {
+ list_remove_head(&blck_nblist, lpfc_ncmd,
+ struct lpfc_nvme_buf, list);
+ if (status) {
+ /* failure, put on abort nvme list */
+ lpfc_ncmd->exch_busy = 1;
+ } else {
+ /* success, put on NVME buffer list */
+ lpfc_ncmd->exch_busy = 0;
+ lpfc_ncmd->status = IOSTAT_SUCCESS;
+ num_posted++;
+ }
+ list_add_tail(&lpfc_ncmd->list, &nvme_nblist);
+ }
+ }
+ /* Push NVME buffers with sgl posted to the available list */
+ while (!list_empty(&nvme_nblist)) {
+ list_remove_head(&nvme_nblist, lpfc_ncmd,
+ struct lpfc_nvme_buf, list);
+ lpfc_release_nvme_buf(phba, lpfc_ncmd);
+ }
+ return num_posted;
+}
+
+/**
+ * lpfc_repost_nvme_sgl_list - Repost all the allocated nvme buffer sgls
+ * @phba: pointer to lpfc hba data structure.
+ *
+ * This routine walks the list of nvme buffers that have been allocated and
+ * repost them to the port by using SGL block post. This is needed after a
+ * pci_function_reset/warm_start or start. The lpfc_hba_down_post_s4 routine
+ * is responsible for moving all nvme buffers on the lpfc_abts_nvme_sgl_list
+ * to the lpfc_nvme_buf_list. If the repost fails, reject all nvme buffers.
+ *
+ * Returns: 0 = success, non-zero failure.
+ **/
+int
+lpfc_repost_nvme_sgl_list(struct lpfc_hba *phba)
+{
+ LIST_HEAD(post_nblist);
+ int num_posted, rc = 0;
+
+ /* get all NVME buffers need to repost to a local list */
+ spin_lock_irq(&phba->nvme_buf_list_get_lock);
+ spin_lock(&phba->nvme_buf_list_put_lock);
+ list_splice_init(&phba->lpfc_nvme_buf_list_get, &post_nblist);
+ list_splice(&phba->lpfc_nvme_buf_list_put, &post_nblist);
+ spin_unlock(&phba->nvme_buf_list_put_lock);
+ spin_unlock_irq(&phba->nvme_buf_list_get_lock);
+
+ /* post the list of nvme buffer sgls to port if available */
+ if (!list_empty(&post_nblist)) {
+ num_posted = lpfc_post_nvme_sgl_list(phba, &post_nblist,
+ phba->sli4_hba.nvme_xri_cnt);
+ /* failed to post any nvme buffer, return error */
+ if (num_posted == 0)
+ rc = -EIO;
+ }
+ return rc;
+}
+
+/**
+ * lpfc_new_nvme_buf - Scsi buffer allocator for HBA with SLI4 IF spec
+ * @vport: The virtual port for which this call being executed.
+ * @num_to_allocate: The requested number of buffers to allocate.
+ *
+ * This routine allocates nvme buffers for device with SLI-4 interface spec,
+ * the nvme buffer contains all the necessary information needed to initiate
+ * a NVME I/O. After allocating up to @num_to_allocate NVME buffers and put
+ * them on a list, it post them to the port by using SGL block post.
+ *
+ * Return codes:
+ * int - number of nvme buffers that were allocated and posted.
+ * 0 = failure, less than num_to_alloc is a partial failure.
+ **/
+static int
+lpfc_new_nvme_buf(struct lpfc_vport *vport, int num_to_alloc)
+{
+ struct lpfc_hba *phba = vport->phba;
+ struct lpfc_nvme_buf *lpfc_ncmd;
+ struct lpfc_iocbq *pwqeq;
+ union lpfc_wqe128 *wqe;
+ struct sli4_sge *sgl;
+ dma_addr_t pdma_phys_sgl;
+ uint16_t iotag, lxri = 0;
+ int bcnt, num_posted, sgl_size;
+ LIST_HEAD(prep_nblist);
+ LIST_HEAD(post_nblist);
+ LIST_HEAD(nvme_nblist);
+
+ sgl_size = phba->cfg_sg_dma_buf_size;
+
+ for (bcnt = 0; bcnt < num_to_alloc; bcnt++) {
+ lpfc_ncmd = kzalloc(sizeof(struct lpfc_nvme_buf), GFP_KERNEL);
+ if (!lpfc_ncmd)
+ break;
+ /*
+ * Get memory from the pci pool to map the virt space to
+ * pci bus space for an I/O. The DMA buffer includes the
+ * number of SGE's necessary to support the sg_tablesize.
+ */
+ lpfc_ncmd->data = pci_pool_alloc(phba->lpfc_sg_dma_buf_pool,
+ GFP_KERNEL,
+ &lpfc_ncmd->dma_handle);
+ if (!lpfc_ncmd->data) {
+ kfree(lpfc_ncmd);
+ break;
+ }
+ memset(lpfc_ncmd->data, 0, phba->cfg_sg_dma_buf_size);
+
+ lxri = lpfc_sli4_next_xritag(phba);
+ if (lxri == NO_XRI) {
+ pci_pool_free(phba->lpfc_sg_dma_buf_pool,
+ lpfc_ncmd->data, lpfc_ncmd->dma_handle);
+ kfree(lpfc_ncmd);
+ break;
+ }
+ pwqeq = &(lpfc_ncmd->cur_iocbq);
+ wqe = (union lpfc_wqe128 *)&pwqeq->wqe;
+
+ /* Allocate iotag for lpfc_ncmd->cur_iocbq. */
+ iotag = lpfc_sli_next_iotag(phba, pwqeq);
+ if (iotag == 0) {
+ pci_pool_free(phba->lpfc_sg_dma_buf_pool,
+ lpfc_ncmd->data, lpfc_ncmd->dma_handle);
+ kfree(lpfc_ncmd);
+ lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR,
+ "6121 Failed to allocated IOTAG for"
+ " XRI:0x%x\n", lxri);
+ lpfc_sli4_free_xri(phba, lxri);
+ break;
+ }
+ pwqeq->sli4_lxritag = lxri;
+ pwqeq->sli4_xritag = phba->sli4_hba.xri_ids[lxri];
+ pwqeq->iocb_flag |= LPFC_IO_NVME;
+ pwqeq->context1 = lpfc_ncmd;
+ pwqeq->wqe_cmpl = lpfc_nvme_io_cmd_wqe_cmpl;
+
+ /* Initialize local short-hand pointers. */
+ lpfc_ncmd->nvme_sgl = lpfc_ncmd->data;
+ sgl = lpfc_ncmd->nvme_sgl;
+ pdma_phys_sgl = lpfc_ncmd->dma_handle;
+ lpfc_ncmd->dma_phys_sgl = pdma_phys_sgl;
+
+ /* Rsp SGE will be filled in when we rcv an IO
+ * from the NVME Layer to be sent.
+ * The cmd is going to be embedded so we need a SKIP SGE.
+ */
+ bf_set(lpfc_sli4_sge_type, sgl, LPFC_SGE_TYPE_SKIP);
+ bf_set(lpfc_sli4_sge_last, sgl, 0);
+ sgl->word2 = cpu_to_le32(sgl->word2);
+ /* Fill in word 3 / sgl_len during cmd submission */
+
+ lpfc_ncmd->cur_iocbq.context1 = lpfc_ncmd;
+
+ /* Word 7 */
+ bf_set(wqe_erp, &wqe->generic.wqe_com, 0);
+ /* NVME upper layers will time things out, if needed */
+ bf_set(wqe_tmo, &wqe->generic.wqe_com, 0);
+
+ /* Word 10 */
+ bf_set(wqe_ebde_cnt, &wqe->generic.wqe_com, 0);
+ bf_set(wqe_dbde, &wqe->generic.wqe_com, 1);
+
+ /* add the nvme buffer to a post list */
+ list_add_tail(&lpfc_ncmd->list, &post_nblist);
+ spin_lock_irq(&phba->nvme_buf_list_get_lock);
+ phba->sli4_hba.nvme_xri_cnt++;
+ spin_unlock_irq(&phba->nvme_buf_list_get_lock);
+ }
+ lpfc_printf_log(phba, KERN_INFO, LOG_NVME,
+ "6114 Allocate %d out of %d requested new NVME "
+ "buffers\n", bcnt, num_to_alloc);
+
+ /* post the list of nvme buffer sgls to port if available */
+ if (!list_empty(&post_nblist))
+ num_posted = lpfc_post_nvme_sgl_list(phba,
+ &post_nblist, bcnt);
+ else
+ num_posted = 0;
+
+ return num_posted;
+}
+
+/**
+ * lpfc_get_nvme_buf - Get a nvme buffer from lpfc_nvme_buf_list of the HBA
+ * @phba: The HBA for which this call is being executed.
+ *
+ * This routine removes a nvme buffer from head of @phba lpfc_nvme_buf_list list
+ * and returns to caller.
+ *
+ * Return codes:
+ * NULL - Error
+ * Pointer to lpfc_nvme_buf - Success
+ **/
+static struct lpfc_nvme_buf *
+lpfc_get_nvme_buf(struct lpfc_hba *phba, struct lpfc_nodelist *ndlp)
+{
+ struct lpfc_nvme_buf *lpfc_ncmd, *lpfc_ncmd_next;
+ unsigned long iflag = 0;
+ int found = 0;
+
+ spin_lock_irqsave(&phba->nvme_buf_list_get_lock, iflag);
+ list_for_each_entry_safe(lpfc_ncmd, lpfc_ncmd_next,
+ &phba->lpfc_nvme_buf_list_get, list) {
+ if (lpfc_test_rrq_active(phba, ndlp,
+ lpfc_ncmd->cur_iocbq.sli4_lxritag))
+ continue;
+ list_del(&lpfc_ncmd->list);
+ found = 1;
+ break;
+ }
+ if (!found) {
+ spin_lock(&phba->nvme_buf_list_put_lock);
+ list_splice(&phba->lpfc_nvme_buf_list_put,
+ &phba->lpfc_nvme_buf_list_get);
+ INIT_LIST_HEAD(&phba->lpfc_nvme_buf_list_put);
+ spin_unlock(&phba->nvme_buf_list_put_lock);
+ list_for_each_entry_safe(lpfc_ncmd, lpfc_ncmd_next,
+ &phba->lpfc_nvme_buf_list_get, list) {
+ if (lpfc_test_rrq_active(
+ phba, ndlp, lpfc_ncmd->cur_iocbq.sli4_lxritag))
+ continue;
+ list_del(&lpfc_ncmd->list);
+ found = 1;
+ break;
+ }
+ }
+ spin_unlock_irqrestore(&phba->nvme_buf_list_get_lock, iflag);
+ if (!found)
+ return NULL;
+ return lpfc_ncmd;
+}
+
+/**
+ * lpfc_release_nvme_buf: Return a nvme buffer back to hba nvme buf list.
+ * @phba: The Hba for which this call is being executed.
+ * @lpfc_ncmd: The nvme buffer which is being released.
+ *
+ * This routine releases @lpfc_ncmd nvme buffer by adding it to tail of @phba
+ * lpfc_nvme_buf_list list. For SLI4 XRI's are tied to the nvme buffer
+ * and cannot be reused for at least RA_TOV amount of time if it was
+ * aborted.
+ **/
+static void
+lpfc_release_nvme_buf(struct lpfc_hba *phba, struct lpfc_nvme_buf *lpfc_ncmd)
+{
+ unsigned long iflag = 0;
+
+ lpfc_ncmd->nonsg_phys = 0;
+ if (lpfc_ncmd->exch_busy) {
+ spin_lock_irqsave(&phba->sli4_hba.abts_nvme_buf_list_lock,
+ iflag);
+ lpfc_ncmd->nvmeCmd = NULL;
+ list_add_tail(&lpfc_ncmd->list,
+ &phba->sli4_hba.lpfc_abts_nvme_buf_list);
+ spin_unlock_irqrestore(&phba->sli4_hba.abts_nvme_buf_list_lock,
+ iflag);
+ } else {
+ lpfc_ncmd->nvmeCmd = NULL;
+ lpfc_ncmd->cur_iocbq.iocb_flag = LPFC_IO_NVME;
+ spin_lock_irqsave(&phba->nvme_buf_list_put_lock, iflag);
+ list_add_tail(&lpfc_ncmd->list, &phba->lpfc_nvme_buf_list_put);
+ spin_unlock_irqrestore(&phba->nvme_buf_list_put_lock, iflag);
+ }
+}
+
+/**
+ * lpfc_nvme_create_localport - Create/Bind an nvme localport instance.
+ * @pvport - the lpfc_vport instance requesting a localport.
+ *
+ * This routine is invoked to create an nvme localport instance to bind
+ * to the nvme_fc_transport. It is called once during driver load
+ * like lpfc_create_shost after all other services are initialized.
+ * It requires a vport, vpi, and wwns at call time. Other localport
+ * parameters are modified as the driver's FCID and the Fabric WWN
+ * are established.
+ *
+ * Return codes
+ * 0 - successful
+ * -ENOMEM - no heap memory available
+ * other values - from nvme registration upcall
+ **/
+int
+lpfc_nvme_create_localport(struct lpfc_vport *vport)
+{
+ struct lpfc_hba *phba = vport->phba;
+ struct nvme_fc_port_info nfcp_info;
+ struct nvme_fc_local_port *localport;
+ struct lpfc_nvme_lport *lport;
+ int len, ret = 0;
+
+ /* Initialize this localport instance. The vport wwn usage ensures
+ * that NPIV is accounted for.
+ */
+ memset(&nfcp_info, 0, sizeof(struct nvme_fc_port_info));
+ nfcp_info.port_role = FC_PORT_ROLE_NVME_INITIATOR;
+ nfcp_info.node_name = wwn_to_u64(vport->fc_nodename.u.wwn);
+ nfcp_info.port_name = wwn_to_u64(vport->fc_portname.u.wwn);
+
+ /* For now need + 1 to get around NVME transport logic */
+ lpfc_nvme_template.max_sgl_segments = phba->cfg_sg_seg_cnt + 1;
+ lpfc_nvme_template.max_hw_queues = phba->cfg_nvme_io_channel;
+
+ /* localport is allocated from the stack, but the registration
+ * call allocates heap memory as well as the private area.
+ */
+ ret = nvme_fc_register_localport(&nfcp_info, &lpfc_nvme_template,
+ &vport->phba->pcidev->dev, &localport);
+ if (!ret) {
+ lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME | LOG_NVME_DISC,
+ "6005 Successfully registered local "
+ "NVME port num %d, localP %p, private %p, "
+ "sg_seg %d\n",
+ localport->port_num, localport,
+ localport->private,
+ lpfc_nvme_template.max_sgl_segments);
+
+ /* Private is our lport size declared in the template. */
+ lport = (struct lpfc_nvme_lport *)localport->private;
+ vport->localport = localport;
+ lport->vport = vport;
+ INIT_LIST_HEAD(&lport->rport_list);
+ vport->nvmei_support = 1;
+ }
+
+ len = lpfc_new_nvme_buf(vport, phba->sli4_hba.nvme_xri_max);
+ vport->phba->total_nvme_bufs += len;
+ return ret;
+}
+
+/**
+ * lpfc_nvme_destroy_localport - Destroy lpfc_nvme bound to nvme transport.
+ * @pnvme: pointer to lpfc nvme data structure.
+ *
+ * This routine is invoked to destroy all lports bound to the phba.
+ * The lport memory was allocated by the nvme fc transport and is
+ * released there. This routine ensures all rports bound to the
+ * lport have been disconnected.
+ *
+ **/
+void
+lpfc_nvme_destroy_localport(struct lpfc_vport *vport)
+{
+ struct nvme_fc_local_port *localport;
+ struct lpfc_nvme_lport *lport;
+ struct lpfc_nvme_rport *rport = NULL, *rport_next = NULL;
+ int ret;
+
+ if (vport->nvmei_support == 0)
+ return;
+
+ localport = vport->localport;
+ vport->localport = NULL;
+ lport = (struct lpfc_nvme_lport *)localport->private;
+
+ lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME,
+ "6011 Destroying NVME localport %p\n",
+ localport);
+
+ list_for_each_entry_safe(rport, rport_next, &lport->rport_list, list) {
+ /* The last node ref has to get released now before the rport
+ * private memory area is released by the transport.
+ */
+ list_del(&rport->list);
+
+ init_completion(&rport->rport_unreg_done);
+ ret = nvme_fc_unregister_remoteport(rport->remoteport);
+ if (ret)
+ lpfc_printf_vlog(vport, KERN_ERR, LOG_NVME_DISC,
+ "6008 rport fail destroy %x\n", ret);
+ wait_for_completion_timeout(&rport->rport_unreg_done, 5);
+ }
+ /* lport's rport list is clear. Unregister
+ * lport and release resources.
+ */
+ init_completion(&lport->lport_unreg_done);
+ ret = nvme_fc_unregister_localport(localport);
+ wait_for_completion_timeout(&lport->lport_unreg_done, 5);
+
+ /* Regardless of the unregister upcall response, clear
+ * nvmei_support. All rports are unregistered and the
+ * driver will clean up.
+ */
+ vport->nvmei_support = 0;
+ if (ret == 0) {
+ lpfc_printf_vlog(vport,
+ KERN_INFO, LOG_NVME_DISC,
+ "6009 Unregistered lport Success\n");
+ } else {
+ lpfc_printf_vlog(vport,
+ KERN_INFO, LOG_NVME_DISC,
+ "6010 Unregistered lport "
+ "Failed, status x%x\n",
+ ret);
+ }
+}
+
+void
+lpfc_nvme_update_localport(struct lpfc_vport *vport)
+{
+ struct nvme_fc_local_port *localport;
+ struct lpfc_nvme_lport *lport;
+
+ localport = vport->localport;
+ lport = (struct lpfc_nvme_lport *)localport->private;
+
+ lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME,
+ "6012 Update NVME lport %p did x%x\n",
+ localport, vport->fc_myDID);
+
+ localport->port_id = vport->fc_myDID;
+ if (localport->port_id == 0)
+ localport->port_role = FC_PORT_ROLE_NVME_DISCOVERY;
+ else
+ localport->port_role = FC_PORT_ROLE_NVME_INITIATOR;
+
+ lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_DISC,
+ "6030 bound lport %p to DID x%06x\n",
+ lport, localport->port_id);
+
+}
+
+int
+lpfc_nvme_register_port(struct lpfc_vport *vport, struct lpfc_nodelist *ndlp)
+{
+ int ret = 0;
+ struct nvme_fc_local_port *localport;
+ struct lpfc_nvme_lport *lport;
+ struct lpfc_nvme_rport *rport;
+ struct nvme_fc_remote_port *remote_port;
+ struct nvme_fc_port_info rpinfo;
+
+ lpfc_printf_vlog(ndlp->vport, KERN_INFO, LOG_NVME_DISC,
+ "6006 Register NVME PORT. DID x%06x nlptype x%x\n",
+ ndlp->nlp_DID, ndlp->nlp_type);
+
+ localport = vport->localport;
+ lport = (struct lpfc_nvme_lport *)localport->private;
+
+ if (ndlp->nlp_type & (NLP_NVME_TARGET | NLP_NVME_INITIATOR)) {
+
+ /* The driver isn't expecting the rport wwn to change
+ * but it might get a different DID on a different
+ * fabric.
+ */
+ list_for_each_entry(rport, &lport->rport_list, list) {
+ if (rport->remoteport->port_name !=
+ wwn_to_u64(ndlp->nlp_portname.u.wwn))
+ continue;
+ lpfc_printf_vlog(ndlp->vport, KERN_INFO, LOG_NVME_DISC,
+ "6035 lport %p, found matching rport "
+ "at wwpn 0x%llx, Data: x%x x%x x%x "
+ "x%06x\n",
+ lport,
+ rport->remoteport->port_name,
+ rport->remoteport->port_id,
+ rport->remoteport->port_role,
+ ndlp->nlp_type,
+ ndlp->nlp_DID);
+ remote_port = rport->remoteport;
+ if ((remote_port->port_id == 0) &&
+ (remote_port->port_role ==
+ FC_PORT_ROLE_NVME_DISCOVERY)) {
+ remote_port->port_id = ndlp->nlp_DID;
+ remote_port->port_role &=
+ ~FC_PORT_ROLE_NVME_DISCOVERY;
+ if (ndlp->nlp_type & NLP_NVME_TARGET)
+ remote_port->port_role |=
+ FC_PORT_ROLE_NVME_TARGET;
+ if (ndlp->nlp_type & NLP_NVME_INITIATOR)
+ remote_port->port_role |=
+ FC_PORT_ROLE_NVME_INITIATOR;
+
+ lpfc_printf_vlog(ndlp->vport, KERN_INFO,
+ LOG_NVME_DISC,
+ "6014 Rebinding lport to "
+ "rport wwpn 0x%llx, "
+ "Data: x%x x%x x%x x%06x\n",
+ remote_port->port_name,
+ remote_port->port_id,
+ remote_port->port_role,
+ ndlp->nlp_type,
+ ndlp->nlp_DID);
+ }
+ return 0;
+ }
+
+ /* NVME rports are not preserved across devloss.
+ * Just register this instance.
+ */
+ rpinfo.port_id = ndlp->nlp_DID;
+ rpinfo.port_role = 0;
+ if (ndlp->nlp_type & NLP_NVME_TARGET)
+ rpinfo.port_role |= FC_PORT_ROLE_NVME_TARGET;
+ if (ndlp->nlp_type & NLP_NVME_INITIATOR)
+ rpinfo.port_role |= FC_PORT_ROLE_NVME_INITIATOR;
+ rpinfo.port_name = wwn_to_u64(ndlp->nlp_portname.u.wwn);
+ rpinfo.node_name = wwn_to_u64(ndlp->nlp_nodename.u.wwn);
+
+ ret = nvme_fc_register_remoteport(localport, &rpinfo,
+ &remote_port);
+ if (!ret) {
+ rport = remote_port->private;
+ rport->remoteport = remote_port;
+ rport->lport = lport;
+ rport->ndlp = lpfc_nlp_get(ndlp);
+ if (!rport->ndlp)
+ return -1;
+ ndlp->nrport = rport;
+ INIT_LIST_HEAD(&rport->list);
+ list_add_tail(&rport->list, &lport->rport_list);
+ lpfc_printf_vlog(vport, KERN_INFO,
+ LOG_NVME_DISC | LOG_NODE,
+ "6022 Binding new rport to lport %p "
+ "Rport WWNN 0x%llx, Rport WWPN 0x%llx "
+ "DID x%06x Role x%x\n",
+ lport,
+ rpinfo.node_name, rpinfo.port_name,
+ rpinfo.port_id, rpinfo.port_role);
+ } else {
+ lpfc_printf_vlog(vport, KERN_ERR,
+ LOG_NVME_DISC | LOG_NODE,
+ "6031 RemotePort Registration failed "
+ "err: %d, DID x%06x\n",
+ ret, ndlp->nlp_DID);
+ }
+ } else {
+ ret = -EINVAL;
+ lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_DISC,
+ "6027 Unknown nlp_type x%x on DID x%06x "
+ "ndlp %p. Not Registering nvme rport\n",
+ ndlp->nlp_type, ndlp->nlp_DID, ndlp);
+ }
+ return ret;
+}
+
+/* lpfc_nvme_unregister_port - unbind the DID and port_role from this rport.
+ *
+ * There is no notion of Devloss or rport recovery from the current
+ * nvme_transport perspective. Loss of an rport just means IO cannot
+ * be sent and recovery is completely up to the initator.
+ * For now, the driver just unbinds the DID and port_role so that
+ * no further IO can be issued. Changes are planned for later.
+ *
+ * Notes - the ndlp reference count is not decremented here since
+ * since there is no nvme_transport api for devloss. Node ref count
+ * is only adjusted in driver unload.
+ */
+void
+lpfc_nvme_unregister_port(struct lpfc_vport *vport, struct lpfc_nodelist *ndlp)
+{
+ int ret;
+ struct nvme_fc_local_port *localport;
+ struct lpfc_nvme_lport *lport;
+ struct lpfc_nvme_rport *rport;
+ struct nvme_fc_remote_port *remoteport;
+
+ localport = vport->localport;
+
+ /* This is fundamental error. The localport is always
+ * available until driver unload. Just exit.
+ */
+ if (!localport)
+ return;
+
+ lport = (struct lpfc_nvme_lport *)localport->private;
+ if (!lport)
+ goto input_err;
+
+ rport = ndlp->nrport;
+ if (!rport)
+ goto input_err;
+
+ remoteport = rport->remoteport;
+ lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_DISC,
+ "6033 Unreg nvme remoteport %p, portname x%llx, "
+ "port_id x%06x, portstate x%x port type x%x\n",
+ remoteport, remoteport->port_name,
+ remoteport->port_id, remoteport->port_state,
+ ndlp->nlp_type);
+
+ /* Sanity check ndlp type. Only call for NVME ports. Don't
+ * clear any rport state until the transport calls back.
+ */
+ if (ndlp->nlp_type & (NLP_NVME_TARGET | NLP_NVME_INITIATOR)) {
+ init_completion(&rport->rport_unreg_done);
+ ret = nvme_fc_unregister_remoteport(remoteport);
+ if (ret != 0) {
+ lpfc_printf_vlog(vport, KERN_ERR, LOG_NVME_DISC,
+ "6167 NVME unregister failed %d "
+ "port_state x%x\n",
+ ret, remoteport->port_state);
+ }
+
+ /* Wait for the driver's delete completion routine to finish
+ * before proceeding. This guarantees the transport and driver
+ * have completed the unreg process.
+ */
+ ret = wait_for_completion_timeout(&rport->rport_unreg_done, 5);
+ if (ret == 0) {
+ lpfc_printf_vlog(vport, KERN_ERR, LOG_NVME_DISC,
+ "6169 Unreg nvme wait failed %d\n",
+ ret);
+ }
+ }
+ return;
+
+ input_err:
+ lpfc_printf_vlog(vport, KERN_ERR, LOG_NVME_DISC,
+ "6168: State error: lport %p, rport%p FCID x%06x\n",
+ vport->localport, ndlp->rport, ndlp->nlp_DID);
+}
--- /dev/null
+/*******************************************************************
+ * This file is part of the Emulex Linux Device Driver for *
+ * Fibre Channel Host Bus Adapters. *
+ * Copyright (C) 2017 Broadcom. All Rights Reserved. The term *
+ * “Broadcom” refers to Broadcom Limited and/or its subsidiaries. *
+ * Copyright (C) 2004-2016 Emulex. All rights reserved. *
+ * EMULEX and SLI are trademarks of Emulex. *
+ * www.broadcom.com *
+ * Portions Copyright (C) 2004-2005 Christoph Hellwig *
+ * *
+ * This program is free software; you can redistribute it and/or *
+ * modify it under the terms of version 2 of the GNU General *
+ * Public License as published by the Free Software Foundation. *
+ * This program is distributed in the hope that it will be useful. *
+ * ALL EXPRESS OR IMPLIED CONDITIONS, REPRESENTATIONS AND *
+ * WARRANTIES, INCLUDING ANY IMPLIED WARRANTY OF MERCHANTABILITY, *
+ * FITNESS FOR A PARTICULAR PURPOSE, OR NON-INFRINGEMENT, ARE *
+ * DISCLAIMED, EXCEPT TO THE EXTENT THAT SUCH DISCLAIMERS ARE HELD *
+ * TO BE LEGALLY INVALID. See the GNU General Public License for *
+ * more details, a copy of which can be found in the file COPYING *
+ * included with this package. *
+ ********************************************************************/
+
+#define LPFC_NVME_MIN_SEGS 16
+#define LPFC_NVME_DEFAULT_SEGS 66 /* 256K IOs - 64 + 2 */
+#define LPFC_NVME_MAX_SEGS 510
+#define LPFC_NVMET_MIN_POSTBUF 16
+#define LPFC_NVMET_DEFAULT_POSTBUF 1024
+#define LPFC_NVMET_MAX_POSTBUF 4096
+#define LPFC_NVME_WQSIZE 256
+
+#define LPFC_NVME_ERSP_LEN 0x20
+
+struct lpfc_nvme_qhandle {
+ uint32_t index; /* WQ index to use */
+ uint32_t qidx; /* queue index passed to create */
+ uint32_t cpu_id; /* current cpu id at time of create */
+};
+
+/* Declare nvme-based local and remote port definitions. */
+struct lpfc_nvme_lport {
+ struct lpfc_vport *vport;
+ struct list_head rport_list;
+ struct completion lport_unreg_done;
+ /* Add sttats counters here */
+};
+
+struct lpfc_nvme_rport {
+ struct list_head list;
+ struct lpfc_nvme_lport *lport;
+ struct nvme_fc_remote_port *remoteport;
+ struct lpfc_nodelist *ndlp;
+ struct completion rport_unreg_done;
+};
+
+struct lpfc_nvme_buf {
+ struct list_head list;
+ struct nvmefc_fcp_req *nvmeCmd;
+ struct lpfc_nvme_rport *nrport;
+
+ uint32_t timeout;
+
+ uint16_t flags; /* TBD convert exch_busy to flags */
+#define LPFC_SBUF_XBUSY 0x1 /* SLI4 hba reported XB on WCQE cmpl */
+ uint16_t exch_busy; /* SLI4 hba reported XB on complete WCQE */
+ uint16_t status; /* From IOCB Word 7- ulpStatus */
+ uint16_t cpu;
+ uint16_t qidx;
+ uint16_t sqid;
+ uint32_t result; /* From IOCB Word 4. */
+
+ uint32_t seg_cnt; /* Number of scatter-gather segments returned by
+ * dma_map_sg. The driver needs this for calls
+ * to dma_unmap_sg.
+ */
+ dma_addr_t nonsg_phys; /* Non scatter-gather physical address. */
+
+ /*
+ * data and dma_handle are the kernel virtual and bus address of the
+ * dma-able buffer containing the fcp_cmd, fcp_rsp and a scatter
+ * gather bde list that supports the sg_tablesize value.
+ */
+ void *data;
+ dma_addr_t dma_handle;
+
+ struct sli4_sge *nvme_sgl;
+ dma_addr_t dma_phys_sgl;
+
+ /* cur_iocbq has phys of the dma-able buffer.
+ * Iotag is in here
+ */
+ struct lpfc_iocbq cur_iocbq;
+
+ wait_queue_head_t *waitq;
+ unsigned long start_time;
+#ifdef CONFIG_SCSI_LPFC_DEBUG_FS
+ uint64_t ts_cmd_start;
+ uint64_t ts_last_cmd;
+ uint64_t ts_cmd_wqput;
+ uint64_t ts_isr_cmpl;
+ uint64_t ts_data_nvme;
+#endif
+};
--- /dev/null
+/*******************************************************************
+ * This file is part of the Emulex Linux Device Driver for *
+ * Fibre Channsel Host Bus Adapters. *
+ * Copyright (C) 2017 Broadcom. All Rights Reserved. The term *
+ * “Broadcom” refers to Broadcom Limited and/or its subsidiaries. *
+ * Copyright (C) 2004-2016 Emulex. All rights reserved. *
+ * EMULEX and SLI are trademarks of Emulex. *
+ * www.broadcom.com *
+ * Portions Copyright (C) 2004-2005 Christoph Hellwig *
+ * *
+ * This program is free software; you can redistribute it and/or *
+ * modify it under the terms of version 2 of the GNU General *
+ * Public License as published by the Free Software Foundation. *
+ * This program is distributed in the hope that it will be useful. *
+ * ALL EXPRESS OR IMPLIED CONDITIONS, REPRESENTATIONS AND *
+ * WARRANTIES, INCLUDING ANY IMPLIED WARRANTY OF MERCHANTABILITY, *
+ * FITNESS FOR A PARTICULAR PURPOSE, OR NON-INFRINGEMENT, ARE *
+ * DISCLAIMED, EXCEPT TO THE EXTENT THAT SUCH DISCLAIMERS ARE HELD *
+ * TO BE LEGALLY INVALID. See the GNU General Public License for *
+ * more details, a copy of which can be found in the file COPYING *
+ * included with this package. *
+ ********************************************************************/
+#include <linux/pci.h>
+#include <linux/slab.h>
+#include <linux/interrupt.h>
+#include <linux/delay.h>
+#include <asm/unaligned.h>
+#include <linux/crc-t10dif.h>
+#include <net/checksum.h>
+
+#include <scsi/scsi.h>
+#include <scsi/scsi_device.h>
+#include <scsi/scsi_eh.h>
+#include <scsi/scsi_host.h>
+#include <scsi/scsi_tcq.h>
+#include <scsi/scsi_transport_fc.h>
+#include <scsi/fc/fc_fs.h>
+
+#include <../drivers/nvme/host/nvme.h>
+#include <linux/nvme-fc-driver.h>
+
+#include "lpfc_version.h"
+#include "lpfc_hw4.h"
+#include "lpfc_hw.h"
+#include "lpfc_sli.h"
+#include "lpfc_sli4.h"
+#include "lpfc_nl.h"
+#include "lpfc_disc.h"
+#include "lpfc.h"
+#include "lpfc_scsi.h"
+#include "lpfc_nvme.h"
+#include "lpfc_nvmet.h"
+#include "lpfc_logmsg.h"
+#include "lpfc_crtn.h"
+#include "lpfc_vport.h"
+#include "lpfc_debugfs.h"
+
+static struct lpfc_iocbq *lpfc_nvmet_prep_ls_wqe(struct lpfc_hba *,
+ struct lpfc_nvmet_rcv_ctx *,
+ dma_addr_t rspbuf,
+ uint16_t rspsize);
+static struct lpfc_iocbq *lpfc_nvmet_prep_fcp_wqe(struct lpfc_hba *,
+ struct lpfc_nvmet_rcv_ctx *);
+static int lpfc_nvmet_sol_fcp_issue_abort(struct lpfc_hba *,
+ struct lpfc_nvmet_rcv_ctx *,
+ uint32_t, uint16_t);
+static int lpfc_nvmet_unsol_fcp_issue_abort(struct lpfc_hba *,
+ struct lpfc_nvmet_rcv_ctx *,
+ uint32_t, uint16_t);
+static int lpfc_nvmet_unsol_ls_issue_abort(struct lpfc_hba *,
+ struct lpfc_nvmet_rcv_ctx *,
+ uint32_t, uint16_t);
+
+/**
+ * lpfc_nvmet_xmt_ls_rsp_cmp - Completion handler for LS Response
+ * @phba: Pointer to HBA context object.
+ * @cmdwqe: Pointer to driver command WQE object.
+ * @wcqe: Pointer to driver response CQE object.
+ *
+ * The function is called from SLI ring event handler with no
+ * lock held. This function is the completion handler for NVME LS commands
+ * The function frees memory resources used for the NVME commands.
+ **/
+static void
+lpfc_nvmet_xmt_ls_rsp_cmp(struct lpfc_hba *phba, struct lpfc_iocbq *cmdwqe,
+ struct lpfc_wcqe_complete *wcqe)
+{
+ struct lpfc_nvmet_tgtport *tgtp;
+ struct nvmefc_tgt_ls_req *rsp;
+ struct lpfc_nvmet_rcv_ctx *ctxp;
+ uint32_t status, result;
+
+ status = bf_get(lpfc_wcqe_c_status, wcqe);
+ result = wcqe->parameter;
+ if (!phba->targetport)
+ goto out;
+
+ tgtp = (struct lpfc_nvmet_tgtport *)phba->targetport->private;
+
+ if (status)
+ atomic_inc(&tgtp->xmt_ls_rsp_error);
+ else
+ atomic_inc(&tgtp->xmt_ls_rsp_cmpl);
+
+out:
+ ctxp = cmdwqe->context2;
+ rsp = &ctxp->ctx.ls_req;
+
+ lpfc_nvmeio_data(phba, "NVMET LS CMPL: xri x%x stat x%x result x%x\n",
+ ctxp->oxid, status, result);
+
+ lpfc_printf_log(phba, KERN_INFO, LOG_NVME_DISC,
+ "6038 %s: Entrypoint: ctx %p status %x/%x\n", __func__,
+ ctxp, status, result);
+
+ lpfc_nlp_put(cmdwqe->context1);
+ cmdwqe->context2 = NULL;
+ cmdwqe->context3 = NULL;
+ lpfc_sli_release_iocbq(phba, cmdwqe);
+ rsp->done(rsp);
+ kfree(ctxp);
+}
+
+/**
+ * lpfc_nvmet_rq_post - Repost a NVMET RQ DMA buffer and clean up context
+ * @phba: HBA buffer is associated with
+ * @ctxp: context to clean up
+ * @mp: Buffer to free
+ *
+ * Description: Frees the given DMA buffer in the appropriate way given by
+ * reposting it to its associated RQ so it can be reused.
+ *
+ * Notes: Takes phba->hbalock. Can be called with or without other locks held.
+ *
+ * Returns: None
+ **/
+void
+lpfc_nvmet_rq_post(struct lpfc_hba *phba, struct lpfc_nvmet_rcv_ctx *ctxp,
+ struct lpfc_dmabuf *mp)
+{
+ if (ctxp) {
+ if (ctxp->txrdy) {
+ pci_pool_free(phba->txrdy_payload_pool, ctxp->txrdy,
+ ctxp->txrdy_phys);
+ ctxp->txrdy = NULL;
+ ctxp->txrdy_phys = 0;
+ }
+ ctxp->state = LPFC_NVMET_STE_FREE;
+ }
+ lpfc_rq_buf_free(phba, mp);
+}
+
+#ifdef CONFIG_SCSI_LPFC_DEBUG_FS
+static void
+lpfc_nvmet_ktime(struct lpfc_hba *phba,
+ struct lpfc_nvmet_rcv_ctx *ctxp)
+{
+ uint64_t seg1, seg2, seg3, seg4, seg5;
+ uint64_t seg6, seg7, seg8, seg9, seg10;
+
+ if (!phba->ktime_on)
+ return;
+
+ if (!ctxp->ts_isr_cmd || !ctxp->ts_cmd_nvme ||
+ !ctxp->ts_nvme_data || !ctxp->ts_data_wqput ||
+ !ctxp->ts_isr_data || !ctxp->ts_data_nvme ||
+ !ctxp->ts_nvme_status || !ctxp->ts_status_wqput ||
+ !ctxp->ts_isr_status || !ctxp->ts_status_nvme)
+ return;
+
+ if (ctxp->ts_isr_cmd > ctxp->ts_cmd_nvme)
+ return;
+ if (ctxp->ts_cmd_nvme > ctxp->ts_nvme_data)
+ return;
+ if (ctxp->ts_nvme_data > ctxp->ts_data_wqput)
+ return;
+ if (ctxp->ts_data_wqput > ctxp->ts_isr_data)
+ return;
+ if (ctxp->ts_isr_data > ctxp->ts_data_nvme)
+ return;
+ if (ctxp->ts_data_nvme > ctxp->ts_nvme_status)
+ return;
+ if (ctxp->ts_nvme_status > ctxp->ts_status_wqput)
+ return;
+ if (ctxp->ts_status_wqput > ctxp->ts_isr_status)
+ return;
+ if (ctxp->ts_isr_status > ctxp->ts_status_nvme)
+ return;
+ /*
+ * Segment 1 - Time from FCP command received by MSI-X ISR
+ * to FCP command is passed to NVME Layer.
+ * Segment 2 - Time from FCP command payload handed
+ * off to NVME Layer to Driver receives a Command op
+ * from NVME Layer.
+ * Segment 3 - Time from Driver receives a Command op
+ * from NVME Layer to Command is put on WQ.
+ * Segment 4 - Time from Driver WQ put is done
+ * to MSI-X ISR for Command cmpl.
+ * Segment 5 - Time from MSI-X ISR for Command cmpl to
+ * Command cmpl is passed to NVME Layer.
+ * Segment 6 - Time from Command cmpl is passed to NVME
+ * Layer to Driver receives a RSP op from NVME Layer.
+ * Segment 7 - Time from Driver receives a RSP op from
+ * NVME Layer to WQ put is done on TRSP FCP Status.
+ * Segment 8 - Time from Driver WQ put is done on TRSP
+ * FCP Status to MSI-X ISR for TRSP cmpl.
+ * Segment 9 - Time from MSI-X ISR for TRSP cmpl to
+ * TRSP cmpl is passed to NVME Layer.
+ * Segment 10 - Time from FCP command received by
+ * MSI-X ISR to command is completed on wire.
+ * (Segments 1 thru 8) for READDATA / WRITEDATA
+ * (Segments 1 thru 4) for READDATA_RSP
+ */
+ seg1 = ctxp->ts_cmd_nvme - ctxp->ts_isr_cmd;
+ seg2 = (ctxp->ts_nvme_data - ctxp->ts_isr_cmd) - seg1;
+ seg3 = (ctxp->ts_data_wqput - ctxp->ts_isr_cmd) -
+ seg1 - seg2;
+ seg4 = (ctxp->ts_isr_data - ctxp->ts_isr_cmd) -
+ seg1 - seg2 - seg3;
+ seg5 = (ctxp->ts_data_nvme - ctxp->ts_isr_cmd) -
+ seg1 - seg2 - seg3 - seg4;
+
+ /* For auto rsp commands seg6 thru seg10 will be 0 */
+ if (ctxp->ts_nvme_status > ctxp->ts_data_nvme) {
+ seg6 = (ctxp->ts_nvme_status -
+ ctxp->ts_isr_cmd) -
+ seg1 - seg2 - seg3 - seg4 - seg5;
+ seg7 = (ctxp->ts_status_wqput -
+ ctxp->ts_isr_cmd) -
+ seg1 - seg2 - seg3 -
+ seg4 - seg5 - seg6;
+ seg8 = (ctxp->ts_isr_status -
+ ctxp->ts_isr_cmd) -
+ seg1 - seg2 - seg3 - seg4 -
+ seg5 - seg6 - seg7;
+ seg9 = (ctxp->ts_status_nvme -
+ ctxp->ts_isr_cmd) -
+ seg1 - seg2 - seg3 - seg4 -
+ seg5 - seg6 - seg7 - seg8;
+ seg10 = (ctxp->ts_isr_status -
+ ctxp->ts_isr_cmd);
+ } else {
+ seg6 = 0;
+ seg7 = 0;
+ seg8 = 0;
+ seg9 = 0;
+ seg10 = (ctxp->ts_isr_data - ctxp->ts_isr_cmd);
+ }
+
+ phba->ktime_seg1_total += seg1;
+ if (seg1 < phba->ktime_seg1_min)
+ phba->ktime_seg1_min = seg1;
+ else if (seg1 > phba->ktime_seg1_max)
+ phba->ktime_seg1_max = seg1;
+
+ phba->ktime_seg2_total += seg2;
+ if (seg2 < phba->ktime_seg2_min)
+ phba->ktime_seg2_min = seg2;
+ else if (seg2 > phba->ktime_seg2_max)
+ phba->ktime_seg2_max = seg2;
+
+ phba->ktime_seg3_total += seg3;
+ if (seg3 < phba->ktime_seg3_min)
+ phba->ktime_seg3_min = seg3;
+ else if (seg3 > phba->ktime_seg3_max)
+ phba->ktime_seg3_max = seg3;
+
+ phba->ktime_seg4_total += seg4;
+ if (seg4 < phba->ktime_seg4_min)
+ phba->ktime_seg4_min = seg4;
+ else if (seg4 > phba->ktime_seg4_max)
+ phba->ktime_seg4_max = seg4;
+
+ phba->ktime_seg5_total += seg5;
+ if (seg5 < phba->ktime_seg5_min)
+ phba->ktime_seg5_min = seg5;
+ else if (seg5 > phba->ktime_seg5_max)
+ phba->ktime_seg5_max = seg5;
+
+ phba->ktime_data_samples++;
+ if (!seg6)
+ goto out;
+
+ phba->ktime_seg6_total += seg6;
+ if (seg6 < phba->ktime_seg6_min)
+ phba->ktime_seg6_min = seg6;
+ else if (seg6 > phba->ktime_seg6_max)
+ phba->ktime_seg6_max = seg6;
+
+ phba->ktime_seg7_total += seg7;
+ if (seg7 < phba->ktime_seg7_min)
+ phba->ktime_seg7_min = seg7;
+ else if (seg7 > phba->ktime_seg7_max)
+ phba->ktime_seg7_max = seg7;
+
+ phba->ktime_seg8_total += seg8;
+ if (seg8 < phba->ktime_seg8_min)
+ phba->ktime_seg8_min = seg8;
+ else if (seg8 > phba->ktime_seg8_max)
+ phba->ktime_seg8_max = seg8;
+
+ phba->ktime_seg9_total += seg9;
+ if (seg9 < phba->ktime_seg9_min)
+ phba->ktime_seg9_min = seg9;
+ else if (seg9 > phba->ktime_seg9_max)
+ phba->ktime_seg9_max = seg9;
+out:
+ phba->ktime_seg10_total += seg10;
+ if (seg10 < phba->ktime_seg10_min)
+ phba->ktime_seg10_min = seg10;
+ else if (seg10 > phba->ktime_seg10_max)
+ phba->ktime_seg10_max = seg10;
+ phba->ktime_status_samples++;
+}
+#endif
+
+/**
+ * lpfc_nvmet_xmt_fcp_op_cmp - Completion handler for FCP Response
+ * @phba: Pointer to HBA context object.
+ * @cmdwqe: Pointer to driver command WQE object.
+ * @wcqe: Pointer to driver response CQE object.
+ *
+ * The function is called from SLI ring event handler with no
+ * lock held. This function is the completion handler for NVME FCP commands
+ * The function frees memory resources used for the NVME commands.
+ **/
+static void
+lpfc_nvmet_xmt_fcp_op_cmp(struct lpfc_hba *phba, struct lpfc_iocbq *cmdwqe,
+ struct lpfc_wcqe_complete *wcqe)
+{
+ struct lpfc_nvmet_tgtport *tgtp;
+ struct nvmefc_tgt_fcp_req *rsp;
+ struct lpfc_nvmet_rcv_ctx *ctxp;
+ uint32_t status, result, op, start_clean;
+#ifdef CONFIG_SCSI_LPFC_DEBUG_FS
+ uint32_t id;
+#endif
+
+ ctxp = cmdwqe->context2;
+ rsp = &ctxp->ctx.fcp_req;
+ op = rsp->op;
+ ctxp->flag &= ~LPFC_NVMET_IO_INP;
+
+ status = bf_get(lpfc_wcqe_c_status, wcqe);
+ result = wcqe->parameter;
+
+ if (!phba->targetport)
+ goto out;
+
+ lpfc_nvmeio_data(phba, "NVMET FCP CMPL: xri x%x op x%x status x%x\n",
+ ctxp->oxid, op, status);
+
+ tgtp = (struct lpfc_nvmet_tgtport *)phba->targetport->private;
+ if (status) {
+ rsp->fcp_error = NVME_SC_DATA_XFER_ERROR;
+ rsp->transferred_length = 0;
+ atomic_inc(&tgtp->xmt_fcp_rsp_error);
+ } else {
+ rsp->fcp_error = NVME_SC_SUCCESS;
+ if (op == NVMET_FCOP_RSP)
+ rsp->transferred_length = rsp->rsplen;
+ else
+ rsp->transferred_length = rsp->transfer_length;
+ atomic_inc(&tgtp->xmt_fcp_rsp_cmpl);
+ }
+
+out:
+ if ((op == NVMET_FCOP_READDATA_RSP) ||
+ (op == NVMET_FCOP_RSP)) {
+ /* Sanity check */
+ ctxp->state = LPFC_NVMET_STE_DONE;
+ ctxp->entry_cnt++;
+#ifdef CONFIG_SCSI_LPFC_DEBUG_FS
+ if (phba->ktime_on) {
+ if (rsp->op == NVMET_FCOP_READDATA_RSP) {
+ ctxp->ts_isr_data =
+ cmdwqe->isr_timestamp;
+ ctxp->ts_data_nvme =
+ ktime_get_ns();
+ ctxp->ts_nvme_status =
+ ctxp->ts_data_nvme;
+ ctxp->ts_status_wqput =
+ ctxp->ts_data_nvme;
+ ctxp->ts_isr_status =
+ ctxp->ts_data_nvme;
+ ctxp->ts_status_nvme =
+ ctxp->ts_data_nvme;
+ } else {
+ ctxp->ts_isr_status =
+ cmdwqe->isr_timestamp;
+ ctxp->ts_status_nvme =
+ ktime_get_ns();
+ }
+ }
+ if (phba->cpucheck_on & LPFC_CHECK_NVMET_IO) {
+ id = smp_processor_id();
+ if (ctxp->cpu != id)
+ lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR,
+ "6703 CPU Check cmpl: "
+ "cpu %d expect %d\n",
+ id, ctxp->cpu);
+ if (ctxp->cpu < LPFC_CHECK_CPU_CNT)
+ phba->cpucheck_cmpl_io[id]++;
+ }
+#endif
+ rsp->done(rsp);
+#ifdef CONFIG_SCSI_LPFC_DEBUG_FS
+ if (phba->ktime_on)
+ lpfc_nvmet_ktime(phba, ctxp);
+#endif
+ /* Let Abort cmpl repost the context */
+ if (!(ctxp->flag & LPFC_NVMET_ABORT_OP))
+ lpfc_nvmet_rq_post(phba, ctxp, &ctxp->rqb_buffer->hbuf);
+ } else {
+ ctxp->entry_cnt++;
+ start_clean = offsetof(struct lpfc_iocbq, wqe);
+ memset(((char *)cmdwqe) + start_clean, 0,
+ (sizeof(struct lpfc_iocbq) - start_clean));
+#ifdef CONFIG_SCSI_LPFC_DEBUG_FS
+ if (phba->ktime_on) {
+ ctxp->ts_isr_data = cmdwqe->isr_timestamp;
+ ctxp->ts_data_nvme = ktime_get_ns();
+ }
+ if (phba->cpucheck_on & LPFC_CHECK_NVMET_IO) {
+ id = smp_processor_id();
+ if (ctxp->cpu != id)
+ lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR,
+ "6704 CPU Check cmdcmpl: "
+ "cpu %d expect %d\n",
+ id, ctxp->cpu);
+ if (ctxp->cpu < LPFC_CHECK_CPU_CNT)
+ phba->cpucheck_ccmpl_io[id]++;
+ }
+#endif
+ rsp->done(rsp);
+ }
+}
+
+static int
+lpfc_nvmet_xmt_ls_rsp(struct nvmet_fc_target_port *tgtport,
+ struct nvmefc_tgt_ls_req *rsp)
+{
+ struct lpfc_nvmet_rcv_ctx *ctxp =
+ container_of(rsp, struct lpfc_nvmet_rcv_ctx, ctx.ls_req);
+ struct lpfc_hba *phba = ctxp->phba;
+ struct hbq_dmabuf *nvmebuf =
+ (struct hbq_dmabuf *)ctxp->rqb_buffer;
+ struct lpfc_iocbq *nvmewqeq;
+ struct lpfc_nvmet_tgtport *nvmep = tgtport->private;
+ struct lpfc_dmabuf dmabuf;
+ struct ulp_bde64 bpl;
+ int rc;
+
+ lpfc_printf_log(phba, KERN_INFO, LOG_NVME_DISC,
+ "6023 %s: Entrypoint ctx %p %p\n", __func__,
+ ctxp, tgtport);
+
+ nvmewqeq = lpfc_nvmet_prep_ls_wqe(phba, ctxp, rsp->rspdma,
+ rsp->rsplen);
+ if (nvmewqeq == NULL) {
+ atomic_inc(&nvmep->xmt_ls_drop);
+ lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR,
+ "6150 LS Drop IO x%x: Prep\n",
+ ctxp->oxid);
+ lpfc_in_buf_free(phba, &nvmebuf->dbuf);
+ lpfc_nvmet_unsol_ls_issue_abort(phba, ctxp,
+ ctxp->sid, ctxp->oxid);
+ return -ENOMEM;
+ }
+
+ /* Save numBdes for bpl2sgl */
+ nvmewqeq->rsvd2 = 1;
+ nvmewqeq->hba_wqidx = 0;
+ nvmewqeq->context3 = &dmabuf;
+ dmabuf.virt = &bpl;
+ bpl.addrLow = nvmewqeq->wqe.xmit_sequence.bde.addrLow;
+ bpl.addrHigh = nvmewqeq->wqe.xmit_sequence.bde.addrHigh;
+ bpl.tus.f.bdeSize = rsp->rsplen;
+ bpl.tus.f.bdeFlags = 0;
+ bpl.tus.w = le32_to_cpu(bpl.tus.w);
+
+ nvmewqeq->wqe_cmpl = lpfc_nvmet_xmt_ls_rsp_cmp;
+ nvmewqeq->iocb_cmpl = NULL;
+ nvmewqeq->context2 = ctxp;
+
+ lpfc_nvmeio_data(phba, "NVMET LS RESP: xri x%x wqidx x%x len x%x\n",
+ ctxp->oxid, nvmewqeq->hba_wqidx, rsp->rsplen);
+
+ rc = lpfc_sli4_issue_wqe(phba, LPFC_ELS_RING, nvmewqeq);
+ if (rc == WQE_SUCCESS) {
+ /*
+ * Okay to repost buffer here, but wait till cmpl
+ * before freeing ctxp and iocbq.
+ */
+ lpfc_in_buf_free(phba, &nvmebuf->dbuf);
+ ctxp->rqb_buffer = 0;
+ atomic_inc(&nvmep->xmt_ls_rsp);
+ return 0;
+ }
+ /* Give back resources */
+ atomic_inc(&nvmep->xmt_ls_drop);
+ lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR,
+ "6151 LS Drop IO x%x: Issue %d\n",
+ ctxp->oxid, rc);
+
+ lpfc_nlp_put(nvmewqeq->context1);
+
+ lpfc_in_buf_free(phba, &nvmebuf->dbuf);
+ lpfc_nvmet_unsol_ls_issue_abort(phba, ctxp, ctxp->sid, ctxp->oxid);
+ return -ENXIO;
+}
+
+static int
+lpfc_nvmet_xmt_fcp_op(struct nvmet_fc_target_port *tgtport,
+ struct nvmefc_tgt_fcp_req *rsp)
+{
+ struct lpfc_nvmet_tgtport *lpfc_nvmep = tgtport->private;
+ struct lpfc_nvmet_rcv_ctx *ctxp =
+ container_of(rsp, struct lpfc_nvmet_rcv_ctx, ctx.fcp_req);
+ struct lpfc_hba *phba = ctxp->phba;
+ struct lpfc_iocbq *nvmewqeq;
+ unsigned long iflags;
+ int rc, id;
+
+#ifdef CONFIG_SCSI_LPFC_DEBUG_FS
+ if (phba->ktime_on) {
+ if (rsp->op == NVMET_FCOP_RSP)
+ ctxp->ts_nvme_status = ktime_get_ns();
+ else
+ ctxp->ts_nvme_data = ktime_get_ns();
+ }
+ if (phba->cpucheck_on & LPFC_CHECK_NVMET_IO) {
+ id = smp_processor_id();
+ ctxp->cpu = id;
+ if (id < LPFC_CHECK_CPU_CNT)
+ phba->cpucheck_xmt_io[id]++;
+ if (rsp->hwqid != id) {
+ lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR,
+ "6705 CPU Check OP: "
+ "cpu %d expect %d\n",
+ id, rsp->hwqid);
+ ctxp->cpu = rsp->hwqid;
+ }
+ }
+#endif
+
+ if (rsp->op == NVMET_FCOP_ABORT) {
+ lpfc_printf_log(phba, KERN_INFO, LOG_NVME_ABTS,
+ "6103 Abort op: oxri x%x %d cnt %d\n",
+ ctxp->oxid, ctxp->state, ctxp->entry_cnt);
+
+ lpfc_nvmeio_data(phba, "NVMET FCP ABRT: "
+ "xri x%x state x%x cnt x%x\n",
+ ctxp->oxid, ctxp->state, ctxp->entry_cnt);
+
+ atomic_inc(&lpfc_nvmep->xmt_fcp_abort);
+ ctxp->entry_cnt++;
+ ctxp->flag |= LPFC_NVMET_ABORT_OP;
+ if (ctxp->flag & LPFC_NVMET_IO_INP)
+ lpfc_nvmet_sol_fcp_issue_abort(phba, ctxp, ctxp->sid,
+ ctxp->oxid);
+ else
+ lpfc_nvmet_unsol_fcp_issue_abort(phba, ctxp, ctxp->sid,
+ ctxp->oxid);
+ return 0;
+ }
+
+ /* Sanity check */
+ if (ctxp->state == LPFC_NVMET_STE_ABORT) {
+ atomic_inc(&lpfc_nvmep->xmt_fcp_drop);
+ lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR,
+ "6102 Bad state IO x%x aborted\n",
+ ctxp->oxid);
+ goto aerr;
+ }
+
+ nvmewqeq = lpfc_nvmet_prep_fcp_wqe(phba, ctxp);
+ if (nvmewqeq == NULL) {
+ atomic_inc(&lpfc_nvmep->xmt_fcp_drop);
+ lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR,
+ "6152 FCP Drop IO x%x: Prep\n",
+ ctxp->oxid);
+ goto aerr;
+ }
+
+ nvmewqeq->wqe_cmpl = lpfc_nvmet_xmt_fcp_op_cmp;
+ nvmewqeq->iocb_cmpl = NULL;
+ nvmewqeq->context2 = ctxp;
+ nvmewqeq->iocb_flag |= LPFC_IO_NVMET;
+ ctxp->wqeq->hba_wqidx = rsp->hwqid;
+
+ lpfc_nvmeio_data(phba, "NVMET FCP CMND: xri x%x op x%x len x%x\n",
+ ctxp->oxid, rsp->op, rsp->rsplen);
+
+ /* For now we take hbalock */
+ spin_lock_irqsave(&phba->hbalock, iflags);
+ rc = lpfc_sli4_issue_wqe(phba, LPFC_FCP_RING, nvmewqeq);
+ spin_unlock_irqrestore(&phba->hbalock, iflags);
+ if (rc == WQE_SUCCESS) {
+ ctxp->flag |= LPFC_NVMET_IO_INP;
+#ifdef CONFIG_SCSI_LPFC_DEBUG_FS
+ if (!phba->ktime_on)
+ return 0;
+ if (rsp->op == NVMET_FCOP_RSP)
+ ctxp->ts_status_wqput = ktime_get_ns();
+ else
+ ctxp->ts_data_wqput = ktime_get_ns();
+#endif
+ return 0;
+ }
+
+ /* Give back resources */
+ atomic_inc(&lpfc_nvmep->xmt_fcp_drop);
+ lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR,
+ "6153 FCP Drop IO x%x: Issue: %d\n",
+ ctxp->oxid, rc);
+
+ ctxp->wqeq->hba_wqidx = 0;
+ nvmewqeq->context2 = NULL;
+ nvmewqeq->context3 = NULL;
+aerr:
+ return -ENXIO;
+}
+
+static void
+lpfc_nvmet_targetport_delete(struct nvmet_fc_target_port *targetport)
+{
+ struct lpfc_nvmet_tgtport *tport = targetport->private;
+
+ /* release any threads waiting for the unreg to complete */
+ complete(&tport->tport_unreg_done);
+}
+
+static struct nvmet_fc_target_template lpfc_tgttemplate = {
+ .targetport_delete = lpfc_nvmet_targetport_delete,
+ .xmt_ls_rsp = lpfc_nvmet_xmt_ls_rsp,
+ .fcp_op = lpfc_nvmet_xmt_fcp_op,
+
+ .max_hw_queues = 1,
+ .max_sgl_segments = LPFC_NVMET_DEFAULT_SEGS,
+ .max_dif_sgl_segments = LPFC_NVMET_DEFAULT_SEGS,
+ .dma_boundary = 0xFFFFFFFF,
+
+ /* optional features */
+ .target_features = 0,
+ /* sizes of additional private data for data structures */
+ .target_priv_sz = sizeof(struct lpfc_nvmet_tgtport),
+};
+
+int
+lpfc_nvmet_create_targetport(struct lpfc_hba *phba)
+{
+ struct lpfc_vport *vport = phba->pport;
+ struct lpfc_nvmet_tgtport *tgtp;
+ struct nvmet_fc_port_info pinfo;
+ int error = 0;
+
+ if (phba->targetport)
+ return 0;
+
+ memset(&pinfo, 0, sizeof(struct nvmet_fc_port_info));
+ pinfo.node_name = wwn_to_u64(vport->fc_nodename.u.wwn);
+ pinfo.port_name = wwn_to_u64(vport->fc_portname.u.wwn);
+ pinfo.port_id = vport->fc_myDID;
+
+ lpfc_tgttemplate.max_hw_queues = phba->cfg_nvme_io_channel;
+ lpfc_tgttemplate.max_sgl_segments = phba->cfg_sg_seg_cnt;
+ lpfc_tgttemplate.target_features = NVMET_FCTGTFEAT_READDATA_RSP |
+ NVMET_FCTGTFEAT_NEEDS_CMD_CPUSCHED;
+
+ error = nvmet_fc_register_targetport(&pinfo, &lpfc_tgttemplate,
+ &phba->pcidev->dev,
+ &phba->targetport);
+ if (error) {
+ lpfc_printf_log(phba, KERN_ERR, LOG_NVME_DISC,
+ "6025 Cannot register NVME targetport "
+ "x%x\n", error);
+ phba->targetport = NULL;
+ } else {
+ tgtp = (struct lpfc_nvmet_tgtport *)
+ phba->targetport->private;
+ tgtp->phba = phba;
+
+ lpfc_printf_log(phba, KERN_INFO, LOG_NVME_DISC,
+ "6026 Registered NVME "
+ "targetport: %p, private %p "
+ "portnm %llx nodenm %llx\n",
+ phba->targetport, tgtp,
+ pinfo.port_name, pinfo.node_name);
+
+ atomic_set(&tgtp->rcv_ls_req_in, 0);
+ atomic_set(&tgtp->rcv_ls_req_out, 0);
+ atomic_set(&tgtp->rcv_ls_req_drop, 0);
+ atomic_set(&tgtp->xmt_ls_abort, 0);
+ atomic_set(&tgtp->xmt_ls_rsp, 0);
+ atomic_set(&tgtp->xmt_ls_drop, 0);
+ atomic_set(&tgtp->xmt_ls_rsp_error, 0);
+ atomic_set(&tgtp->xmt_ls_rsp_cmpl, 0);
+ atomic_set(&tgtp->rcv_fcp_cmd_in, 0);
+ atomic_set(&tgtp->rcv_fcp_cmd_out, 0);
+ atomic_set(&tgtp->rcv_fcp_cmd_drop, 0);
+ atomic_set(&tgtp->xmt_fcp_abort, 0);
+ atomic_set(&tgtp->xmt_fcp_drop, 0);
+ atomic_set(&tgtp->xmt_fcp_read_rsp, 0);
+ atomic_set(&tgtp->xmt_fcp_read, 0);
+ atomic_set(&tgtp->xmt_fcp_write, 0);
+ atomic_set(&tgtp->xmt_fcp_rsp, 0);
+ atomic_set(&tgtp->xmt_fcp_rsp_cmpl, 0);
+ atomic_set(&tgtp->xmt_fcp_rsp_error, 0);
+ atomic_set(&tgtp->xmt_fcp_rsp_drop, 0);
+ atomic_set(&tgtp->xmt_abort_rsp, 0);
+ atomic_set(&tgtp->xmt_abort_rsp_error, 0);
+ atomic_set(&tgtp->xmt_abort_cmpl, 0);
+ }
+ return error;
+}
+
+int
+lpfc_nvmet_update_targetport(struct lpfc_hba *phba)
+{
+ struct lpfc_vport *vport = phba->pport;
+
+ if (!phba->targetport)
+ return 0;
+
+ lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME,
+ "6007 Update NVMET port %p did x%x\n",
+ phba->targetport, vport->fc_myDID);
+
+ phba->targetport->port_id = vport->fc_myDID;
+ return 0;
+}
+
+void
+lpfc_nvmet_destroy_targetport(struct lpfc_hba *phba)
+{
+ struct lpfc_nvmet_tgtport *tgtp;
+
+ if (phba->nvmet_support == 0)
+ return;
+ if (phba->targetport) {
+ tgtp = (struct lpfc_nvmet_tgtport *)phba->targetport->private;
+ init_completion(&tgtp->tport_unreg_done);
+ nvmet_fc_unregister_targetport(phba->targetport);
+ wait_for_completion_timeout(&tgtp->tport_unreg_done, 5);
+ }
+ phba->targetport = NULL;
+}
+
+/**
+ * lpfc_nvmet_unsol_ls_buffer - Process an unsolicited event data buffer
+ * @phba: pointer to lpfc hba data structure.
+ * @pring: pointer to a SLI ring.
+ * @nvmebuf: pointer to lpfc nvme command HBQ data structure.
+ *
+ * This routine is used for processing the WQE associated with a unsolicited
+ * event. It first determines whether there is an existing ndlp that matches
+ * the DID from the unsolicited WQE. If not, it will create a new one with
+ * the DID from the unsolicited WQE. The ELS command from the unsolicited
+ * WQE is then used to invoke the proper routine and to set up proper state
+ * of the discovery state machine.
+ **/
+static void
+lpfc_nvmet_unsol_ls_buffer(struct lpfc_hba *phba, struct lpfc_sli_ring *pring,
+ struct hbq_dmabuf *nvmebuf)
+{
+ struct lpfc_nvmet_tgtport *tgtp;
+ struct fc_frame_header *fc_hdr;
+ struct lpfc_nvmet_rcv_ctx *ctxp;
+ uint32_t *payload;
+ uint32_t size, oxid, sid, rc;
+
+ if (!nvmebuf || !phba->targetport) {
+ lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR,
+ "6154 LS Drop IO\n");
+ oxid = 0;
+ size = 0;
+ sid = 0;
+ goto dropit;
+ }
+
+ tgtp = (struct lpfc_nvmet_tgtport *)phba->targetport->private;
+ payload = (uint32_t *)(nvmebuf->dbuf.virt);
+ fc_hdr = (struct fc_frame_header *)(nvmebuf->hbuf.virt);
+ size = bf_get(lpfc_rcqe_length, &nvmebuf->cq_event.cqe.rcqe_cmpl);
+ oxid = be16_to_cpu(fc_hdr->fh_ox_id);
+ sid = sli4_sid_from_fc_hdr(fc_hdr);
+
+ ctxp = kzalloc(sizeof(struct lpfc_nvmet_rcv_ctx), GFP_ATOMIC);
+ if (ctxp == NULL) {
+ atomic_inc(&tgtp->rcv_ls_req_drop);
+ lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR,
+ "6155 LS Drop IO x%x: Alloc\n",
+ oxid);
+dropit:
+ lpfc_nvmeio_data(phba, "NVMET LS DROP: "
+ "xri x%x sz %d from %06x\n",
+ oxid, size, sid);
+ if (nvmebuf)
+ lpfc_in_buf_free(phba, &nvmebuf->dbuf);
+ return;
+ }
+ ctxp->phba = phba;
+ ctxp->size = size;
+ ctxp->oxid = oxid;
+ ctxp->sid = sid;
+ ctxp->wqeq = NULL;
+ ctxp->state = LPFC_NVMET_STE_RCV;
+ ctxp->rqb_buffer = (void *)nvmebuf;
+
+ lpfc_nvmeio_data(phba, "NVMET LS RCV: xri x%x sz %d from %06x\n",
+ oxid, size, sid);
+ /*
+ * The calling sequence should be:
+ * nvmet_fc_rcv_ls_req -> lpfc_nvmet_xmt_ls_rsp/cmp ->_req->done
+ * lpfc_nvmet_xmt_ls_rsp_cmp should free the allocated ctxp.
+ */
+ atomic_inc(&tgtp->rcv_ls_req_in);
+ rc = nvmet_fc_rcv_ls_req(phba->targetport, &ctxp->ctx.ls_req,
+ payload, size);
+
+ lpfc_printf_log(phba, KERN_INFO, LOG_NVME_DISC,
+ "6037 %s: ctx %p sz %d rc %d: %08x %08x %08x "
+ "%08x %08x %08x\n", __func__, ctxp, size, rc,
+ *payload, *(payload+1), *(payload+2),
+ *(payload+3), *(payload+4), *(payload+5));
+
+ if (rc == 0) {
+ atomic_inc(&tgtp->rcv_ls_req_out);
+ return;
+ }
+
+ lpfc_nvmeio_data(phba, "NVMET LS DROP: xri x%x sz %d from %06x\n",
+ oxid, size, sid);
+
+ atomic_inc(&tgtp->rcv_ls_req_drop);
+ lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR,
+ "6156 LS Drop IO x%x: nvmet_fc_rcv_ls_req %d\n",
+ ctxp->oxid, rc);
+
+ /* We assume a rcv'ed cmd ALWAYs fits into 1 buffer */
+ if (nvmebuf)
+ lpfc_in_buf_free(phba, &nvmebuf->dbuf);
+
+ atomic_inc(&tgtp->xmt_ls_abort);
+ lpfc_nvmet_unsol_ls_issue_abort(phba, ctxp, sid, oxid);
+}
+
+/**
+ * lpfc_nvmet_unsol_fcp_buffer - Process an unsolicited event data buffer
+ * @phba: pointer to lpfc hba data structure.
+ * @pring: pointer to a SLI ring.
+ * @nvmebuf: pointer to lpfc nvme command HBQ data structure.
+ *
+ * This routine is used for processing the WQE associated with a unsolicited
+ * event. It first determines whether there is an existing ndlp that matches
+ * the DID from the unsolicited WQE. If not, it will create a new one with
+ * the DID from the unsolicited WQE. The ELS command from the unsolicited
+ * WQE is then used to invoke the proper routine and to set up proper state
+ * of the discovery state machine.
+ **/
+static void
+lpfc_nvmet_unsol_fcp_buffer(struct lpfc_hba *phba,
+ struct lpfc_sli_ring *pring,
+ struct rqb_dmabuf *nvmebuf,
+ uint64_t isr_timestamp)
+{
+ struct lpfc_nvmet_rcv_ctx *ctxp;
+ struct lpfc_nvmet_tgtport *tgtp;
+ struct fc_frame_header *fc_hdr;
+ uint32_t *payload;
+ uint32_t size, oxid, sid, rc;
+#ifdef CONFIG_SCSI_LPFC_DEBUG_FS
+ uint32_t id;
+#endif
+
+ if (!nvmebuf || !phba->targetport) {
+ lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR,
+ "6157 FCP Drop IO\n");
+ oxid = 0;
+ size = 0;
+ sid = 0;
+ goto dropit;
+ }
+
+
+ tgtp = (struct lpfc_nvmet_tgtport *)phba->targetport->private;
+ payload = (uint32_t *)(nvmebuf->dbuf.virt);
+ fc_hdr = (struct fc_frame_header *)(nvmebuf->hbuf.virt);
+ size = nvmebuf->bytes_recv;
+ oxid = be16_to_cpu(fc_hdr->fh_ox_id);
+ sid = sli4_sid_from_fc_hdr(fc_hdr);
+
+ ctxp = (struct lpfc_nvmet_rcv_ctx *)nvmebuf->context;
+ if (ctxp == NULL) {
+ atomic_inc(&tgtp->rcv_fcp_cmd_drop);
+ lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR,
+ "6158 FCP Drop IO x%x: Alloc\n",
+ oxid);
+ lpfc_nvmet_rq_post(phba, NULL, &nvmebuf->hbuf);
+ /* Cannot send ABTS without context */
+ return;
+ }
+ memset(ctxp, 0, sizeof(ctxp->ctx));
+ ctxp->wqeq = NULL;
+ ctxp->txrdy = NULL;
+ ctxp->offset = 0;
+ ctxp->phba = phba;
+ ctxp->size = size;
+ ctxp->oxid = oxid;
+ ctxp->sid = sid;
+ ctxp->state = LPFC_NVMET_STE_RCV;
+ ctxp->rqb_buffer = nvmebuf;
+ ctxp->entry_cnt = 1;
+ ctxp->flag = 0;
+
+#ifdef CONFIG_SCSI_LPFC_DEBUG_FS
+ if (phba->ktime_on) {
+ ctxp->ts_isr_cmd = isr_timestamp;
+ ctxp->ts_cmd_nvme = ktime_get_ns();
+ ctxp->ts_nvme_data = 0;
+ ctxp->ts_data_wqput = 0;
+ ctxp->ts_isr_data = 0;
+ ctxp->ts_data_nvme = 0;
+ ctxp->ts_nvme_status = 0;
+ ctxp->ts_status_wqput = 0;
+ ctxp->ts_isr_status = 0;
+ ctxp->ts_status_nvme = 0;
+ }
+
+ if (phba->cpucheck_on & LPFC_CHECK_NVMET_RCV) {
+ id = smp_processor_id();
+ if (id < LPFC_CHECK_CPU_CNT)
+ phba->cpucheck_rcv_io[id]++;
+ }
+#endif
+
+ lpfc_nvmeio_data(phba, "NVMET FCP RCV: xri x%x sz %d from %06x\n",
+ oxid, size, sid);
+
+ atomic_inc(&tgtp->rcv_fcp_cmd_in);
+ /*
+ * The calling sequence should be:
+ * nvmet_fc_rcv_fcp_req -> lpfc_nvmet_xmt_fcp_op/cmp -> req->done
+ * lpfc_nvmet_xmt_fcp_op_cmp should free the allocated ctxp.
+ */
+ rc = nvmet_fc_rcv_fcp_req(phba->targetport, &ctxp->ctx.fcp_req,
+ payload, size);
+
+ /* Process FCP command */
+ if (rc == 0) {
+ atomic_inc(&tgtp->rcv_fcp_cmd_out);
+ return;
+ }
+
+ atomic_inc(&tgtp->rcv_fcp_cmd_drop);
+ lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR,
+ "6159 FCP Drop IO x%x: nvmet_fc_rcv_fcp_req x%x\n",
+ ctxp->oxid, rc);
+dropit:
+ lpfc_nvmeio_data(phba, "NVMET FCP DROP: xri x%x sz %d from %06x\n",
+ oxid, size, sid);
+ if (oxid) {
+ lpfc_nvmet_unsol_fcp_issue_abort(phba, ctxp, sid, oxid);
+ return;
+ }
+
+ if (nvmebuf) {
+ nvmebuf->iocbq->hba_wqidx = 0;
+ /* We assume a rcv'ed cmd ALWAYs fits into 1 buffer */
+ lpfc_nvmet_rq_post(phba, NULL, &nvmebuf->hbuf);
+ }
+}
+
+/**
+ * lpfc_nvmet_unsol_ls_event - Process an unsolicited event from an nvme nport
+ * @phba: pointer to lpfc hba data structure.
+ * @pring: pointer to a SLI ring.
+ * @nvmebuf: pointer to received nvme data structure.
+ *
+ * This routine is used to process an unsolicited event received from a SLI
+ * (Service Level Interface) ring. The actual processing of the data buffer
+ * associated with the unsolicited event is done by invoking the routine
+ * lpfc_nvmet_unsol_ls_buffer() after properly set up the buffer from the
+ * SLI RQ on which the unsolicited event was received.
+ **/
+void
+lpfc_nvmet_unsol_ls_event(struct lpfc_hba *phba, struct lpfc_sli_ring *pring,
+ struct lpfc_iocbq *piocb)
+{
+ struct lpfc_dmabuf *d_buf;
+ struct hbq_dmabuf *nvmebuf;
+
+ d_buf = piocb->context2;
+ nvmebuf = container_of(d_buf, struct hbq_dmabuf, dbuf);
+
+ if (phba->nvmet_support == 0) {
+ lpfc_in_buf_free(phba, &nvmebuf->dbuf);
+ return;
+ }
+ lpfc_nvmet_unsol_ls_buffer(phba, pring, nvmebuf);
+}
+
+/**
+ * lpfc_nvmet_unsol_fcp_event - Process an unsolicited event from an nvme nport
+ * @phba: pointer to lpfc hba data structure.
+ * @pring: pointer to a SLI ring.
+ * @nvmebuf: pointer to received nvme data structure.
+ *
+ * This routine is used to process an unsolicited event received from a SLI
+ * (Service Level Interface) ring. The actual processing of the data buffer
+ * associated with the unsolicited event is done by invoking the routine
+ * lpfc_nvmet_unsol_fcp_buffer() after properly set up the buffer from the
+ * SLI RQ on which the unsolicited event was received.
+ **/
+void
+lpfc_nvmet_unsol_fcp_event(struct lpfc_hba *phba,
+ struct lpfc_sli_ring *pring,
+ struct rqb_dmabuf *nvmebuf,
+ uint64_t isr_timestamp)
+{
+ if (phba->nvmet_support == 0) {
+ lpfc_nvmet_rq_post(phba, NULL, &nvmebuf->hbuf);
+ return;
+ }
+ lpfc_nvmet_unsol_fcp_buffer(phba, pring, nvmebuf,
+ isr_timestamp);
+}
+
+/**
+ * lpfc_nvmet_prep_ls_wqe - Allocate and prepare a lpfc wqe data structure
+ * @phba: pointer to a host N_Port data structure.
+ * @ctxp: Context info for NVME LS Request
+ * @rspbuf: DMA buffer of NVME command.
+ * @rspsize: size of the NVME command.
+ *
+ * This routine is used for allocating a lpfc-WQE data structure from
+ * the driver lpfc-WQE free-list and prepare the WQE with the parameters
+ * passed into the routine for discovery state machine to issue an Extended
+ * Link Service (NVME) commands. It is a generic lpfc-WQE allocation
+ * and preparation routine that is used by all the discovery state machine
+ * routines and the NVME command-specific fields will be later set up by
+ * the individual discovery machine routines after calling this routine
+ * allocating and preparing a generic WQE data structure. It fills in the
+ * Buffer Descriptor Entries (BDEs), allocates buffers for both command
+ * payload and response payload (if expected). The reference count on the
+ * ndlp is incremented by 1 and the reference to the ndlp is put into
+ * context1 of the WQE data structure for this WQE to hold the ndlp
+ * reference for the command's callback function to access later.
+ *
+ * Return code
+ * Pointer to the newly allocated/prepared nvme wqe data structure
+ * NULL - when nvme wqe data structure allocation/preparation failed
+ **/
+static struct lpfc_iocbq *
+lpfc_nvmet_prep_ls_wqe(struct lpfc_hba *phba,
+ struct lpfc_nvmet_rcv_ctx *ctxp,
+ dma_addr_t rspbuf, uint16_t rspsize)
+{
+ struct lpfc_nodelist *ndlp;
+ struct lpfc_iocbq *nvmewqe;
+ union lpfc_wqe *wqe;
+
+ if (!lpfc_is_link_up(phba)) {
+ lpfc_printf_log(phba, KERN_ERR, LOG_NVME_DISC,
+ "6104 lpfc_nvmet_prep_ls_wqe: link err: "
+ "NPORT x%x oxid:x%x\n",
+ ctxp->sid, ctxp->oxid);
+ return NULL;
+ }
+
+ /* Allocate buffer for command wqe */
+ nvmewqe = lpfc_sli_get_iocbq(phba);
+ if (nvmewqe == NULL) {
+ lpfc_printf_log(phba, KERN_ERR, LOG_NVME_DISC,
+ "6105 lpfc_nvmet_prep_ls_wqe: No WQE: "
+ "NPORT x%x oxid:x%x\n",
+ ctxp->sid, ctxp->oxid);
+ return NULL;
+ }
+
+ ndlp = lpfc_findnode_did(phba->pport, ctxp->sid);
+ if (!ndlp || !NLP_CHK_NODE_ACT(ndlp) ||
+ ((ndlp->nlp_state != NLP_STE_UNMAPPED_NODE) &&
+ (ndlp->nlp_state != NLP_STE_MAPPED_NODE))) {
+ lpfc_printf_log(phba, KERN_ERR, LOG_NVME_DISC,
+ "6106 lpfc_nvmet_prep_ls_wqe: No ndlp: "
+ "NPORT x%x oxid:x%x\n",
+ ctxp->sid, ctxp->oxid);
+ goto nvme_wqe_free_wqeq_exit;
+ }
+ ctxp->wqeq = nvmewqe;
+
+ /* prevent preparing wqe with NULL ndlp reference */
+ nvmewqe->context1 = lpfc_nlp_get(ndlp);
+ if (nvmewqe->context1 == NULL)
+ goto nvme_wqe_free_wqeq_exit;
+ nvmewqe->context2 = ctxp;
+
+ wqe = &nvmewqe->wqe;
+ memset(wqe, 0, sizeof(union lpfc_wqe));
+
+ /* Words 0 - 2 */
+ wqe->xmit_sequence.bde.tus.f.bdeFlags = BUFF_TYPE_BDE_64;
+ wqe->xmit_sequence.bde.tus.f.bdeSize = rspsize;
+ wqe->xmit_sequence.bde.addrLow = le32_to_cpu(putPaddrLow(rspbuf));
+ wqe->xmit_sequence.bde.addrHigh = le32_to_cpu(putPaddrHigh(rspbuf));
+
+ /* Word 3 */
+
+ /* Word 4 */
+
+ /* Word 5 */
+ bf_set(wqe_dfctl, &wqe->xmit_sequence.wge_ctl, 0);
+ bf_set(wqe_ls, &wqe->xmit_sequence.wge_ctl, 1);
+ bf_set(wqe_la, &wqe->xmit_sequence.wge_ctl, 0);
+ bf_set(wqe_rctl, &wqe->xmit_sequence.wge_ctl, FC_RCTL_DD_SOL_CTL);
+ bf_set(wqe_type, &wqe->xmit_sequence.wge_ctl, FC_TYPE_NVME);
+
+ /* Word 6 */
+ bf_set(wqe_ctxt_tag, &wqe->xmit_sequence.wqe_com,
+ phba->sli4_hba.rpi_ids[ndlp->nlp_rpi]);
+ bf_set(wqe_xri_tag, &wqe->xmit_sequence.wqe_com, nvmewqe->sli4_xritag);
+
+ /* Word 7 */
+ bf_set(wqe_cmnd, &wqe->xmit_sequence.wqe_com,
+ CMD_XMIT_SEQUENCE64_WQE);
+ bf_set(wqe_ct, &wqe->xmit_sequence.wqe_com, SLI4_CT_RPI);
+ bf_set(wqe_class, &wqe->xmit_sequence.wqe_com, CLASS3);
+ bf_set(wqe_pu, &wqe->xmit_sequence.wqe_com, 0);
+
+ /* Word 8 */
+ wqe->xmit_sequence.wqe_com.abort_tag = nvmewqe->iotag;
+
+ /* Word 9 */
+ bf_set(wqe_reqtag, &wqe->xmit_sequence.wqe_com, nvmewqe->iotag);
+ /* Needs to be set by caller */
+ bf_set(wqe_rcvoxid, &wqe->xmit_sequence.wqe_com, ctxp->oxid);
+
+ /* Word 10 */
+ bf_set(wqe_dbde, &wqe->xmit_sequence.wqe_com, 1);
+ bf_set(wqe_iod, &wqe->xmit_sequence.wqe_com, LPFC_WQE_IOD_WRITE);
+ bf_set(wqe_lenloc, &wqe->xmit_sequence.wqe_com,
+ LPFC_WQE_LENLOC_WORD12);
+ bf_set(wqe_ebde_cnt, &wqe->xmit_sequence.wqe_com, 0);
+
+ /* Word 11 */
+ bf_set(wqe_cqid, &wqe->xmit_sequence.wqe_com,
+ LPFC_WQE_CQ_ID_DEFAULT);
+ bf_set(wqe_cmd_type, &wqe->xmit_sequence.wqe_com,
+ OTHER_COMMAND);
+
+ /* Word 12 */
+ wqe->xmit_sequence.xmit_len = rspsize;
+
+ nvmewqe->retry = 1;
+ nvmewqe->vport = phba->pport;
+ nvmewqe->drvrTimeout = (phba->fc_ratov * 3) + LPFC_DRVR_TIMEOUT;
+ nvmewqe->iocb_flag |= LPFC_IO_NVME_LS;
+
+ /* Xmit NVME response to remote NPORT <did> */
+ lpfc_printf_log(phba, KERN_INFO, LOG_NVME_DISC,
+ "6039 Xmit NVME LS response to remote "
+ "NPORT x%x iotag:x%x oxid:x%x size:x%x\n",
+ ndlp->nlp_DID, nvmewqe->iotag, ctxp->oxid,
+ rspsize);
+ return nvmewqe;
+
+nvme_wqe_free_wqeq_exit:
+ nvmewqe->context2 = NULL;
+ nvmewqe->context3 = NULL;
+ lpfc_sli_release_iocbq(phba, nvmewqe);
+ return NULL;
+}
+
+
+static struct lpfc_iocbq *
+lpfc_nvmet_prep_fcp_wqe(struct lpfc_hba *phba,
+ struct lpfc_nvmet_rcv_ctx *ctxp)
+{
+ struct nvmefc_tgt_fcp_req *rsp = &ctxp->ctx.fcp_req;
+ struct lpfc_nvmet_tgtport *tgtp;
+ struct sli4_sge *sgl;
+ struct lpfc_nodelist *ndlp;
+ struct lpfc_iocbq *nvmewqe;
+ struct scatterlist *sgel;
+ union lpfc_wqe128 *wqe;
+ uint32_t *txrdy;
+ dma_addr_t physaddr;
+ int i, cnt;
+ int xc = 1;
+
+ if (!lpfc_is_link_up(phba)) {
+ lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR,
+ "6107 lpfc_nvmet_prep_fcp_wqe: link err:"
+ "NPORT x%x oxid:x%x\n", ctxp->sid,
+ ctxp->oxid);
+ return NULL;
+ }
+
+ ndlp = lpfc_findnode_did(phba->pport, ctxp->sid);
+ if (!ndlp || !NLP_CHK_NODE_ACT(ndlp) ||
+ ((ndlp->nlp_state != NLP_STE_UNMAPPED_NODE) &&
+ (ndlp->nlp_state != NLP_STE_MAPPED_NODE))) {
+ lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR,
+ "6108 lpfc_nvmet_prep_fcp_wqe: no ndlp: "
+ "NPORT x%x oxid:x%x\n",
+ ctxp->sid, ctxp->oxid);
+ return NULL;
+ }
+
+ if (rsp->sg_cnt > phba->cfg_sg_seg_cnt) {
+ lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR,
+ "6109 lpfc_nvmet_prep_fcp_wqe: seg cnt err: "
+ "NPORT x%x oxid:x%x\n",
+ ctxp->sid, ctxp->oxid);
+ return NULL;
+ }
+
+ tgtp = (struct lpfc_nvmet_tgtport *)phba->targetport->private;
+ nvmewqe = ctxp->wqeq;
+ if (nvmewqe == NULL) {
+ /* Allocate buffer for command wqe */
+ nvmewqe = ctxp->rqb_buffer->iocbq;
+ if (nvmewqe == NULL) {
+ lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR,
+ "6110 lpfc_nvmet_prep_fcp_wqe: No "
+ "WQE: NPORT x%x oxid:x%x\n",
+ ctxp->sid, ctxp->oxid);
+ return NULL;
+ }
+ ctxp->wqeq = nvmewqe;
+ xc = 0; /* create new XRI */
+ nvmewqe->sli4_lxritag = NO_XRI;
+ nvmewqe->sli4_xritag = NO_XRI;
+ }
+
+ /* Sanity check */
+ if (((ctxp->state == LPFC_NVMET_STE_RCV) &&
+ (ctxp->entry_cnt == 1)) ||
+ ((ctxp->state == LPFC_NVMET_STE_DATA) &&
+ (ctxp->entry_cnt > 1))) {
+ wqe = (union lpfc_wqe128 *)&nvmewqe->wqe;
+ } else {
+ lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR,
+ "6111 Wrong state %s: %d cnt %d\n",
+ __func__, ctxp->state, ctxp->entry_cnt);
+ return NULL;
+ }
+
+ sgl = (struct sli4_sge *)ctxp->rqb_buffer->sglq->sgl;
+ switch (rsp->op) {
+ case NVMET_FCOP_READDATA:
+ case NVMET_FCOP_READDATA_RSP:
+ /* Words 0 - 2 : The first sg segment */
+ sgel = &rsp->sg[0];
+ physaddr = sg_dma_address(sgel);
+ wqe->fcp_tsend.bde.tus.f.bdeFlags = BUFF_TYPE_BDE_64;
+ wqe->fcp_tsend.bde.tus.f.bdeSize = sg_dma_len(sgel);
+ wqe->fcp_tsend.bde.addrLow = cpu_to_le32(putPaddrLow(physaddr));
+ wqe->fcp_tsend.bde.addrHigh =
+ cpu_to_le32(putPaddrHigh(physaddr));
+
+ /* Word 3 */
+ wqe->fcp_tsend.payload_offset_len = 0;
+
+ /* Word 4 */
+ wqe->fcp_tsend.relative_offset = ctxp->offset;
+
+ /* Word 5 */
+
+ /* Word 6 */
+ bf_set(wqe_ctxt_tag, &wqe->fcp_tsend.wqe_com,
+ phba->sli4_hba.rpi_ids[ndlp->nlp_rpi]);
+ bf_set(wqe_xri_tag, &wqe->fcp_tsend.wqe_com,
+ nvmewqe->sli4_xritag);
+
+ /* Word 7 */
+ bf_set(wqe_cmnd, &wqe->fcp_tsend.wqe_com, CMD_FCP_TSEND64_WQE);
+
+ /* Word 8 */
+ wqe->fcp_tsend.wqe_com.abort_tag = nvmewqe->iotag;
+
+ /* Word 9 */
+ bf_set(wqe_reqtag, &wqe->fcp_tsend.wqe_com, nvmewqe->iotag);
+ bf_set(wqe_rcvoxid, &wqe->fcp_tsend.wqe_com, ctxp->oxid);
+
+ /* Word 10 */
+ bf_set(wqe_nvme, &wqe->fcp_tsend.wqe_com, 1);
+ bf_set(wqe_dbde, &wqe->fcp_tsend.wqe_com, 1);
+ bf_set(wqe_iod, &wqe->fcp_tsend.wqe_com, LPFC_WQE_IOD_WRITE);
+ bf_set(wqe_lenloc, &wqe->fcp_tsend.wqe_com,
+ LPFC_WQE_LENLOC_WORD12);
+ bf_set(wqe_ebde_cnt, &wqe->fcp_tsend.wqe_com, 0);
+ bf_set(wqe_xc, &wqe->fcp_tsend.wqe_com, xc);
+ bf_set(wqe_nvme, &wqe->fcp_tsend.wqe_com, 1);
+ if (phba->cfg_nvme_oas)
+ bf_set(wqe_oas, &wqe->fcp_tsend.wqe_com, 1);
+
+ /* Word 11 */
+ bf_set(wqe_cqid, &wqe->fcp_tsend.wqe_com,
+ LPFC_WQE_CQ_ID_DEFAULT);
+ bf_set(wqe_cmd_type, &wqe->fcp_tsend.wqe_com,
+ FCP_COMMAND_TSEND);
+
+ /* Word 12 */
+ wqe->fcp_tsend.fcp_data_len = rsp->transfer_length;
+
+ /* Setup 2 SKIP SGEs */
+ sgl->addr_hi = 0;
+ sgl->addr_lo = 0;
+ sgl->word2 = 0;
+ bf_set(lpfc_sli4_sge_type, sgl, LPFC_SGE_TYPE_SKIP);
+ sgl->word2 = cpu_to_le32(sgl->word2);
+ sgl->sge_len = 0;
+ sgl++;
+ sgl->addr_hi = 0;
+ sgl->addr_lo = 0;
+ sgl->word2 = 0;
+ bf_set(lpfc_sli4_sge_type, sgl, LPFC_SGE_TYPE_SKIP);
+ sgl->word2 = cpu_to_le32(sgl->word2);
+ sgl->sge_len = 0;
+ sgl++;
+ if (rsp->op == NVMET_FCOP_READDATA_RSP) {
+ atomic_inc(&tgtp->xmt_fcp_read_rsp);
+ bf_set(wqe_ar, &wqe->fcp_tsend.wqe_com, 1);
+ if ((ndlp->nlp_flag & NLP_SUPPRESS_RSP) &&
+ (rsp->rsplen == 12)) {
+ bf_set(wqe_sup, &wqe->fcp_tsend.wqe_com, 1);
+ bf_set(wqe_wqes, &wqe->fcp_tsend.wqe_com, 0);
+ bf_set(wqe_irsp, &wqe->fcp_tsend.wqe_com, 0);
+ bf_set(wqe_irsplen, &wqe->fcp_tsend.wqe_com, 0);
+ } else {
+ bf_set(wqe_sup, &wqe->fcp_tsend.wqe_com, 0);
+ bf_set(wqe_wqes, &wqe->fcp_tsend.wqe_com, 1);
+ bf_set(wqe_irsp, &wqe->fcp_tsend.wqe_com, 1);
+ bf_set(wqe_irsplen, &wqe->fcp_tsend.wqe_com,
+ ((rsp->rsplen >> 2) - 1));
+ memcpy(&wqe->words[16], rsp->rspaddr,
+ rsp->rsplen);
+ }
+ } else {
+ atomic_inc(&tgtp->xmt_fcp_read);
+
+ bf_set(wqe_sup, &wqe->fcp_tsend.wqe_com, 0);
+ bf_set(wqe_wqes, &wqe->fcp_tsend.wqe_com, 0);
+ bf_set(wqe_irsp, &wqe->fcp_tsend.wqe_com, 0);
+ bf_set(wqe_ar, &wqe->fcp_tsend.wqe_com, 0);
+ bf_set(wqe_irsplen, &wqe->fcp_tsend.wqe_com, 0);
+ }
+ ctxp->state = LPFC_NVMET_STE_DATA;
+ break;
+
+ case NVMET_FCOP_WRITEDATA:
+ /* Words 0 - 2 : The first sg segment */
+ txrdy = pci_pool_alloc(phba->txrdy_payload_pool,
+ GFP_KERNEL, &physaddr);
+ if (!txrdy) {
+ lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR,
+ "6041 Bad txrdy buffer: oxid x%x\n",
+ ctxp->oxid);
+ return NULL;
+ }
+ ctxp->txrdy = txrdy;
+ ctxp->txrdy_phys = physaddr;
+ wqe->fcp_treceive.bde.tus.f.bdeFlags = BUFF_TYPE_BDE_64;
+ wqe->fcp_treceive.bde.tus.f.bdeSize = TXRDY_PAYLOAD_LEN;
+ wqe->fcp_treceive.bde.addrLow =
+ cpu_to_le32(putPaddrLow(physaddr));
+ wqe->fcp_treceive.bde.addrHigh =
+ cpu_to_le32(putPaddrHigh(physaddr));
+
+ /* Word 3 */
+ wqe->fcp_treceive.payload_offset_len = TXRDY_PAYLOAD_LEN;
+
+ /* Word 4 */
+ wqe->fcp_treceive.relative_offset = ctxp->offset;
+
+ /* Word 5 */
+
+ /* Word 6 */
+ bf_set(wqe_ctxt_tag, &wqe->fcp_treceive.wqe_com,
+ phba->sli4_hba.rpi_ids[ndlp->nlp_rpi]);
+ bf_set(wqe_xri_tag, &wqe->fcp_treceive.wqe_com,
+ nvmewqe->sli4_xritag);
+
+ /* Word 7 */
+ bf_set(wqe_ar, &wqe->fcp_treceive.wqe_com, 0);
+ bf_set(wqe_cmnd, &wqe->fcp_treceive.wqe_com,
+ CMD_FCP_TRECEIVE64_WQE);
+
+ /* Word 8 */
+ wqe->fcp_treceive.wqe_com.abort_tag = nvmewqe->iotag;
+
+ /* Word 9 */
+ bf_set(wqe_reqtag, &wqe->fcp_treceive.wqe_com, nvmewqe->iotag);
+ bf_set(wqe_rcvoxid, &wqe->fcp_treceive.wqe_com, ctxp->oxid);
+
+ /* Word 10 */
+ bf_set(wqe_nvme, &wqe->fcp_treceive.wqe_com, 1);
+ bf_set(wqe_dbde, &wqe->fcp_treceive.wqe_com, 1);
+ bf_set(wqe_iod, &wqe->fcp_treceive.wqe_com, LPFC_WQE_IOD_READ);
+ bf_set(wqe_lenloc, &wqe->fcp_treceive.wqe_com,
+ LPFC_WQE_LENLOC_WORD12);
+ bf_set(wqe_xc, &wqe->fcp_treceive.wqe_com, xc);
+ bf_set(wqe_wqes, &wqe->fcp_treceive.wqe_com, 0);
+ bf_set(wqe_irsp, &wqe->fcp_treceive.wqe_com, 0);
+ bf_set(wqe_irsplen, &wqe->fcp_treceive.wqe_com, 0);
+ bf_set(wqe_nvme, &wqe->fcp_treceive.wqe_com, 1);
+ if (phba->cfg_nvme_oas)
+ bf_set(wqe_oas, &wqe->fcp_treceive.wqe_com, 1);
+
+ /* Word 11 */
+ bf_set(wqe_cqid, &wqe->fcp_treceive.wqe_com,
+ LPFC_WQE_CQ_ID_DEFAULT);
+ bf_set(wqe_cmd_type, &wqe->fcp_treceive.wqe_com,
+ FCP_COMMAND_TRECEIVE);
+ bf_set(wqe_sup, &wqe->fcp_tsend.wqe_com, 0);
+
+ /* Word 12 */
+ wqe->fcp_tsend.fcp_data_len = rsp->transfer_length;
+
+ /* Setup 1 TXRDY and 1 SKIP SGE */
+ txrdy[0] = 0;
+ txrdy[1] = cpu_to_be32(rsp->transfer_length);
+ txrdy[2] = 0;
+
+ sgl->addr_hi = putPaddrHigh(physaddr);
+ sgl->addr_lo = putPaddrLow(physaddr);
+ sgl->word2 = 0;
+ bf_set(lpfc_sli4_sge_type, sgl, LPFC_SGE_TYPE_DATA);
+ sgl->word2 = cpu_to_le32(sgl->word2);
+ sgl->sge_len = cpu_to_le32(TXRDY_PAYLOAD_LEN);
+ sgl++;
+ sgl->addr_hi = 0;
+ sgl->addr_lo = 0;
+ sgl->word2 = 0;
+ bf_set(lpfc_sli4_sge_type, sgl, LPFC_SGE_TYPE_SKIP);
+ sgl->word2 = cpu_to_le32(sgl->word2);
+ sgl->sge_len = 0;
+ sgl++;
+ ctxp->state = LPFC_NVMET_STE_DATA;
+ atomic_inc(&tgtp->xmt_fcp_write);
+ break;
+
+ case NVMET_FCOP_RSP:
+ /* Words 0 - 2 */
+ sgel = &rsp->sg[0];
+ physaddr = rsp->rspdma;
+ wqe->fcp_trsp.bde.tus.f.bdeFlags = BUFF_TYPE_BDE_64;
+ wqe->fcp_trsp.bde.tus.f.bdeSize = rsp->rsplen;
+ wqe->fcp_trsp.bde.addrLow =
+ cpu_to_le32(putPaddrLow(physaddr));
+ wqe->fcp_trsp.bde.addrHigh =
+ cpu_to_le32(putPaddrHigh(physaddr));
+
+ /* Word 3 */
+ wqe->fcp_trsp.response_len = rsp->rsplen;
+
+ /* Word 4 */
+ wqe->fcp_trsp.rsvd_4_5[0] = 0;
+
+
+ /* Word 5 */
+
+ /* Word 6 */
+ bf_set(wqe_ctxt_tag, &wqe->fcp_trsp.wqe_com,
+ phba->sli4_hba.rpi_ids[ndlp->nlp_rpi]);
+ bf_set(wqe_xri_tag, &wqe->fcp_trsp.wqe_com,
+ nvmewqe->sli4_xritag);
+
+ /* Word 7 */
+ bf_set(wqe_ag, &wqe->fcp_trsp.wqe_com, 1);
+ bf_set(wqe_cmnd, &wqe->fcp_trsp.wqe_com, CMD_FCP_TRSP64_WQE);
+
+ /* Word 8 */
+ wqe->fcp_trsp.wqe_com.abort_tag = nvmewqe->iotag;
+
+ /* Word 9 */
+ bf_set(wqe_reqtag, &wqe->fcp_trsp.wqe_com, nvmewqe->iotag);
+ bf_set(wqe_rcvoxid, &wqe->fcp_trsp.wqe_com, ctxp->oxid);
+
+ /* Word 10 */
+ bf_set(wqe_nvme, &wqe->fcp_trsp.wqe_com, 1);
+ bf_set(wqe_dbde, &wqe->fcp_trsp.wqe_com, 0);
+ bf_set(wqe_iod, &wqe->fcp_trsp.wqe_com, LPFC_WQE_IOD_WRITE);
+ bf_set(wqe_lenloc, &wqe->fcp_trsp.wqe_com,
+ LPFC_WQE_LENLOC_WORD3);
+ bf_set(wqe_xc, &wqe->fcp_trsp.wqe_com, xc);
+ bf_set(wqe_nvme, &wqe->fcp_trsp.wqe_com, 1);
+ if (phba->cfg_nvme_oas)
+ bf_set(wqe_oas, &wqe->fcp_trsp.wqe_com, 1);
+
+ /* Word 11 */
+ bf_set(wqe_cqid, &wqe->fcp_trsp.wqe_com,
+ LPFC_WQE_CQ_ID_DEFAULT);
+ bf_set(wqe_cmd_type, &wqe->fcp_trsp.wqe_com,
+ FCP_COMMAND_TRSP);
+ bf_set(wqe_sup, &wqe->fcp_tsend.wqe_com, 0);
+ ctxp->state = LPFC_NVMET_STE_RSP;
+
+ if (rsp->rsplen == LPFC_NVMET_SUCCESS_LEN) {
+ /* Good response - all zero's on wire */
+ bf_set(wqe_wqes, &wqe->fcp_trsp.wqe_com, 0);
+ bf_set(wqe_irsp, &wqe->fcp_trsp.wqe_com, 0);
+ bf_set(wqe_irsplen, &wqe->fcp_trsp.wqe_com, 0);
+ } else {
+ bf_set(wqe_wqes, &wqe->fcp_trsp.wqe_com, 1);
+ bf_set(wqe_irsp, &wqe->fcp_trsp.wqe_com, 1);
+ bf_set(wqe_irsplen, &wqe->fcp_trsp.wqe_com,
+ ((rsp->rsplen >> 2) - 1));
+ memcpy(&wqe->words[16], rsp->rspaddr, rsp->rsplen);
+ }
+
+ /* Use rspbuf, NOT sg list */
+ rsp->sg_cnt = 0;
+ sgl->word2 = 0;
+ atomic_inc(&tgtp->xmt_fcp_rsp);
+ break;
+
+ default:
+ lpfc_printf_log(phba, KERN_INFO, LOG_NVME_IOERR,
+ "6064 Unknown Rsp Op %d\n",
+ rsp->op);
+ return NULL;
+ }
+
+ nvmewqe->retry = 1;
+ nvmewqe->vport = phba->pport;
+ nvmewqe->drvrTimeout = (phba->fc_ratov * 3) + LPFC_DRVR_TIMEOUT;
+ nvmewqe->context1 = ndlp;
+
+ for (i = 0; i < rsp->sg_cnt; i++) {
+ sgel = &rsp->sg[i];
+ physaddr = sg_dma_address(sgel);
+ cnt = sg_dma_len(sgel);
+ sgl->addr_hi = putPaddrHigh(physaddr);
+ sgl->addr_lo = putPaddrLow(physaddr);
+ sgl->word2 = 0;
+ bf_set(lpfc_sli4_sge_type, sgl, LPFC_SGE_TYPE_DATA);
+ bf_set(lpfc_sli4_sge_offset, sgl, ctxp->offset);
+ if ((i+1) == rsp->sg_cnt)
+ bf_set(lpfc_sli4_sge_last, sgl, 1);
+ sgl->word2 = cpu_to_le32(sgl->word2);
+ sgl->sge_len = cpu_to_le32(cnt);
+ sgl++;
+ ctxp->offset += cnt;
+ }
+ return nvmewqe;
+}
+
+/**
+ * lpfc_nvmet_sol_fcp_abort_cmp - Completion handler for ABTS
+ * @phba: Pointer to HBA context object.
+ * @cmdwqe: Pointer to driver command WQE object.
+ * @wcqe: Pointer to driver response CQE object.
+ *
+ * The function is called from SLI ring event handler with no
+ * lock held. This function is the completion handler for NVME ABTS for FCP cmds
+ * The function frees memory resources used for the NVME commands.
+ **/
+static void
+lpfc_nvmet_sol_fcp_abort_cmp(struct lpfc_hba *phba, struct lpfc_iocbq *cmdwqe,
+ struct lpfc_wcqe_complete *wcqe)
+{
+ struct lpfc_nvmet_rcv_ctx *ctxp;
+ struct lpfc_nvmet_tgtport *tgtp;
+ uint32_t status, result;
+
+ ctxp = cmdwqe->context2;
+ status = bf_get(lpfc_wcqe_c_status, wcqe);
+ result = wcqe->parameter;
+
+ tgtp = (struct lpfc_nvmet_tgtport *)phba->targetport->private;
+ atomic_inc(&tgtp->xmt_abort_cmpl);
+
+ lpfc_printf_log(phba, KERN_ERR, LOG_NVME_ABTS,
+ "6165 Abort cmpl: xri x%x WCQE: %08x %08x %08x %08x\n",
+ ctxp->oxid, wcqe->word0, wcqe->total_data_placed,
+ result, wcqe->word3);
+
+ ctxp->state = LPFC_NVMET_STE_DONE;
+ lpfc_nvmet_rq_post(phba, ctxp, &ctxp->rqb_buffer->hbuf);
+
+ cmdwqe->context2 = NULL;
+ cmdwqe->context3 = NULL;
+ lpfc_sli_release_iocbq(phba, cmdwqe);
+}
+
+/**
+ * lpfc_nvmet_xmt_fcp_abort_cmp - Completion handler for ABTS
+ * @phba: Pointer to HBA context object.
+ * @cmdwqe: Pointer to driver command WQE object.
+ * @wcqe: Pointer to driver response CQE object.
+ *
+ * The function is called from SLI ring event handler with no
+ * lock held. This function is the completion handler for NVME ABTS for FCP cmds
+ * The function frees memory resources used for the NVME commands.
+ **/
+static void
+lpfc_nvmet_xmt_fcp_abort_cmp(struct lpfc_hba *phba, struct lpfc_iocbq *cmdwqe,
+ struct lpfc_wcqe_complete *wcqe)
+{
+ struct lpfc_nvmet_rcv_ctx *ctxp;
+ struct lpfc_nvmet_tgtport *tgtp;
+ uint32_t status, result;
+
+ ctxp = cmdwqe->context2;
+ status = bf_get(lpfc_wcqe_c_status, wcqe);
+ result = wcqe->parameter;
+
+ tgtp = (struct lpfc_nvmet_tgtport *)phba->targetport->private;
+ atomic_inc(&tgtp->xmt_abort_cmpl);
+
+ lpfc_printf_log(phba, KERN_INFO, LOG_NVME_ABTS,
+ "6070 Abort cmpl: ctx %p WCQE: %08x %08x %08x %08x\n",
+ ctxp, wcqe->word0, wcqe->total_data_placed,
+ result, wcqe->word3);
+
+ if (ctxp) {
+ /* Sanity check */
+ if (ctxp->state != LPFC_NVMET_STE_ABORT) {
+ lpfc_printf_log(phba, KERN_ERR, LOG_NVME_ABTS,
+ "6112 ABORT Wrong state:%d oxid x%x\n",
+ ctxp->state, ctxp->oxid);
+ }
+ ctxp->state = LPFC_NVMET_STE_DONE;
+ lpfc_nvmet_rq_post(phba, ctxp, &ctxp->rqb_buffer->hbuf);
+ cmdwqe->context2 = NULL;
+ cmdwqe->context3 = NULL;
+ }
+}
+
+/**
+ * lpfc_nvmet_xmt_ls_abort_cmp - Completion handler for ABTS
+ * @phba: Pointer to HBA context object.
+ * @cmdwqe: Pointer to driver command WQE object.
+ * @wcqe: Pointer to driver response CQE object.
+ *
+ * The function is called from SLI ring event handler with no
+ * lock held. This function is the completion handler for NVME ABTS for LS cmds
+ * The function frees memory resources used for the NVME commands.
+ **/
+static void
+lpfc_nvmet_xmt_ls_abort_cmp(struct lpfc_hba *phba, struct lpfc_iocbq *cmdwqe,
+ struct lpfc_wcqe_complete *wcqe)
+{
+ struct lpfc_nvmet_rcv_ctx *ctxp;
+ struct lpfc_nvmet_tgtport *tgtp;
+ uint32_t status, result;
+
+ ctxp = cmdwqe->context2;
+ status = bf_get(lpfc_wcqe_c_status, wcqe);
+ result = wcqe->parameter;
+
+ tgtp = (struct lpfc_nvmet_tgtport *)phba->targetport->private;
+ atomic_inc(&tgtp->xmt_abort_cmpl);
+
+ lpfc_printf_log(phba, KERN_INFO, LOG_NVME_ABTS,
+ "6083 Abort cmpl: ctx %p WCQE: %08x %08x %08x %08x\n",
+ ctxp, wcqe->word0, wcqe->total_data_placed,
+ result, wcqe->word3);
+
+ if (ctxp) {
+ cmdwqe->context2 = NULL;
+ cmdwqe->context3 = NULL;
+ lpfc_sli_release_iocbq(phba, cmdwqe);
+ kfree(ctxp);
+ } else
+ lpfc_sli_release_iocbq(phba, cmdwqe);
+}
+
+static int
+lpfc_nvmet_unsol_issue_abort(struct lpfc_hba *phba,
+ struct lpfc_nvmet_rcv_ctx *ctxp,
+ uint32_t sid, uint16_t xri)
+{
+ struct lpfc_nvmet_tgtport *tgtp;
+ struct lpfc_iocbq *abts_wqeq;
+ union lpfc_wqe *wqe_abts;
+ struct lpfc_nodelist *ndlp;
+
+ lpfc_printf_log(phba, KERN_INFO, LOG_NVME_ABTS,
+ "6067 %s: Entrypoint: sid %x xri %x\n", __func__,
+ sid, xri);
+
+ tgtp = (struct lpfc_nvmet_tgtport *)phba->targetport->private;
+
+ ndlp = lpfc_findnode_did(phba->pport, sid);
+ if (!ndlp || !NLP_CHK_NODE_ACT(ndlp) ||
+ ((ndlp->nlp_state != NLP_STE_UNMAPPED_NODE) &&
+ (ndlp->nlp_state != NLP_STE_MAPPED_NODE))) {
+ atomic_inc(&tgtp->xmt_abort_rsp_error);
+ lpfc_printf_log(phba, KERN_WARNING, LOG_NVME_ABTS,
+ "6134 Drop ABTS - wrong NDLP state x%x.\n",
+ ndlp->nlp_state);
+
+ /* No failure to an ABTS request. */
+ return 0;
+ }
+
+ abts_wqeq = ctxp->wqeq;
+ wqe_abts = &abts_wqeq->wqe;
+ ctxp->state = LPFC_NVMET_STE_ABORT;
+
+ /*
+ * Since we zero the whole WQE, we need to ensure we set the WQE fields
+ * that were initialized in lpfc_sli4_nvmet_alloc.
+ */
+ memset(wqe_abts, 0, sizeof(union lpfc_wqe));
+
+ /* Word 5 */
+ bf_set(wqe_dfctl, &wqe_abts->xmit_sequence.wge_ctl, 0);
+ bf_set(wqe_ls, &wqe_abts->xmit_sequence.wge_ctl, 1);
+ bf_set(wqe_la, &wqe_abts->xmit_sequence.wge_ctl, 0);
+ bf_set(wqe_rctl, &wqe_abts->xmit_sequence.wge_ctl, FC_RCTL_BA_ABTS);
+ bf_set(wqe_type, &wqe_abts->xmit_sequence.wge_ctl, FC_TYPE_BLS);
+
+ /* Word 6 */
+ bf_set(wqe_ctxt_tag, &wqe_abts->xmit_sequence.wqe_com,
+ phba->sli4_hba.rpi_ids[ndlp->nlp_rpi]);
+ bf_set(wqe_xri_tag, &wqe_abts->xmit_sequence.wqe_com,
+ abts_wqeq->sli4_xritag);
+
+ /* Word 7 */
+ bf_set(wqe_cmnd, &wqe_abts->xmit_sequence.wqe_com,
+ CMD_XMIT_SEQUENCE64_WQE);
+ bf_set(wqe_ct, &wqe_abts->xmit_sequence.wqe_com, SLI4_CT_RPI);
+ bf_set(wqe_class, &wqe_abts->xmit_sequence.wqe_com, CLASS3);
+ bf_set(wqe_pu, &wqe_abts->xmit_sequence.wqe_com, 0);
+
+ /* Word 8 */
+ wqe_abts->xmit_sequence.wqe_com.abort_tag = abts_wqeq->iotag;
+
+ /* Word 9 */
+ bf_set(wqe_reqtag, &wqe_abts->xmit_sequence.wqe_com, abts_wqeq->iotag);
+ /* Needs to be set by caller */
+ bf_set(wqe_rcvoxid, &wqe_abts->xmit_sequence.wqe_com, xri);
+
+ /* Word 10 */
+ bf_set(wqe_dbde, &wqe_abts->xmit_sequence.wqe_com, 1);
+ bf_set(wqe_iod, &wqe_abts->xmit_sequence.wqe_com, LPFC_WQE_IOD_WRITE);
+ bf_set(wqe_lenloc, &wqe_abts->xmit_sequence.wqe_com,
+ LPFC_WQE_LENLOC_WORD12);
+ bf_set(wqe_ebde_cnt, &wqe_abts->xmit_sequence.wqe_com, 0);
+ bf_set(wqe_qosd, &wqe_abts->xmit_sequence.wqe_com, 0);
+
+ /* Word 11 */
+ bf_set(wqe_cqid, &wqe_abts->xmit_sequence.wqe_com,
+ LPFC_WQE_CQ_ID_DEFAULT);
+ bf_set(wqe_cmd_type, &wqe_abts->xmit_sequence.wqe_com,
+ OTHER_COMMAND);
+
+ abts_wqeq->vport = phba->pport;
+ abts_wqeq->context1 = ndlp;
+ abts_wqeq->context2 = ctxp;
+ abts_wqeq->context3 = NULL;
+ abts_wqeq->rsvd2 = 0;
+ /* hba_wqidx should already be setup from command we are aborting */
+ abts_wqeq->iocb.ulpCommand = CMD_XMIT_SEQUENCE64_CR;
+ abts_wqeq->iocb.ulpLe = 1;
+
+ lpfc_printf_log(phba, KERN_INFO, LOG_NVME_ABTS,
+ "6069 Issue ABTS to xri x%x reqtag x%x\n",
+ xri, abts_wqeq->iotag);
+ return 1;
+}
+
+static int
+lpfc_nvmet_sol_fcp_issue_abort(struct lpfc_hba *phba,
+ struct lpfc_nvmet_rcv_ctx *ctxp,
+ uint32_t sid, uint16_t xri)
+{
+ struct lpfc_nvmet_tgtport *tgtp;
+ struct lpfc_iocbq *abts_wqeq;
+ union lpfc_wqe *abts_wqe;
+ struct lpfc_nodelist *ndlp;
+ unsigned long flags;
+ int rc;
+
+ tgtp = (struct lpfc_nvmet_tgtport *)phba->targetport->private;
+ if (!ctxp->wqeq) {
+ ctxp->wqeq = ctxp->rqb_buffer->iocbq;
+ ctxp->wqeq->hba_wqidx = 0;
+ }
+
+ ndlp = lpfc_findnode_did(phba->pport, sid);
+ if (!ndlp || !NLP_CHK_NODE_ACT(ndlp) ||
+ ((ndlp->nlp_state != NLP_STE_UNMAPPED_NODE) &&
+ (ndlp->nlp_state != NLP_STE_MAPPED_NODE))) {
+ atomic_inc(&tgtp->xmt_abort_rsp_error);
+ lpfc_printf_log(phba, KERN_WARNING, LOG_NVME_ABTS,
+ "6160 Drop ABTS - wrong NDLP state x%x.\n",
+ ndlp->nlp_state);
+
+ /* No failure to an ABTS request. */
+ return 0;
+ }
+
+ /* Issue ABTS for this WQE based on iotag */
+ ctxp->abort_wqeq = lpfc_sli_get_iocbq(phba);
+ if (!ctxp->abort_wqeq) {
+ lpfc_printf_log(phba, KERN_WARNING, LOG_NVME_ABTS,
+ "6161 Abort failed: No wqeqs: "
+ "xri: x%x\n", ctxp->oxid);
+ /* No failure to an ABTS request. */
+ return 0;
+ }
+ abts_wqeq = ctxp->abort_wqeq;
+ abts_wqe = &abts_wqeq->wqe;
+ ctxp->state = LPFC_NVMET_STE_ABORT;
+
+ /* Announce entry to new IO submit field. */
+ lpfc_printf_log(phba, KERN_ERR, LOG_NVME_ABTS,
+ "6162 Abort Request to rport DID x%06x "
+ "for xri x%x x%x\n",
+ ctxp->sid, ctxp->oxid, ctxp->wqeq->sli4_xritag);
+
+ /* If the hba is getting reset, this flag is set. It is
+ * cleared when the reset is complete and rings reestablished.
+ */
+ spin_lock_irqsave(&phba->hbalock, flags);
+ /* driver queued commands are in process of being flushed */
+ if (phba->hba_flag & HBA_NVME_IOQ_FLUSH) {
+ spin_unlock_irqrestore(&phba->hbalock, flags);
+ lpfc_printf_log(phba, KERN_ERR, LOG_NVME,
+ "6163 Driver in reset cleanup - flushing "
+ "NVME Req now. hba_flag x%x oxid x%x\n",
+ phba->hba_flag, ctxp->oxid);
+ lpfc_sli_release_iocbq(phba, abts_wqeq);
+ return 0;
+ }
+
+ /* Outstanding abort is in progress */
+ if (abts_wqeq->iocb_flag & LPFC_DRIVER_ABORTED) {
+ spin_unlock_irqrestore(&phba->hbalock, flags);
+ lpfc_printf_log(phba, KERN_ERR, LOG_NVME,
+ "6164 Outstanding NVME I/O Abort Request "
+ "still pending on oxid x%x\n",
+ ctxp->oxid);
+ lpfc_sli_release_iocbq(phba, abts_wqeq);
+ return 0;
+ }
+
+ /* Ready - mark outstanding as aborted by driver. */
+ abts_wqeq->iocb_flag |= LPFC_DRIVER_ABORTED;
+
+ /* WQEs are reused. Clear stale data and set key fields to
+ * zero like ia, iaab, iaar, xri_tag, and ctxt_tag.
+ */
+ memset(abts_wqe, 0, sizeof(union lpfc_wqe));
+
+ /* word 3 */
+ bf_set(abort_cmd_criteria, &abts_wqe->abort_cmd, T_XRI_TAG);
+
+ /* word 7 */
+ bf_set(wqe_ct, &abts_wqe->abort_cmd.wqe_com, 0);
+ bf_set(wqe_cmnd, &abts_wqe->abort_cmd.wqe_com, CMD_ABORT_XRI_CX);
+
+ /* word 8 - tell the FW to abort the IO associated with this
+ * outstanding exchange ID.
+ */
+ abts_wqe->abort_cmd.wqe_com.abort_tag = ctxp->wqeq->sli4_xritag;
+
+ /* word 9 - this is the iotag for the abts_wqe completion. */
+ bf_set(wqe_reqtag, &abts_wqe->abort_cmd.wqe_com,
+ abts_wqeq->iotag);
+
+ /* word 10 */
+ bf_set(wqe_qosd, &abts_wqe->abort_cmd.wqe_com, 1);
+ bf_set(wqe_lenloc, &abts_wqe->abort_cmd.wqe_com, LPFC_WQE_LENLOC_NONE);
+
+ /* word 11 */
+ bf_set(wqe_cmd_type, &abts_wqe->abort_cmd.wqe_com, OTHER_COMMAND);
+ bf_set(wqe_wqec, &abts_wqe->abort_cmd.wqe_com, 1);
+ bf_set(wqe_cqid, &abts_wqe->abort_cmd.wqe_com, LPFC_WQE_CQ_ID_DEFAULT);
+
+ /* ABTS WQE must go to the same WQ as the WQE to be aborted */
+ abts_wqeq->hba_wqidx = ctxp->wqeq->hba_wqidx;
+ abts_wqeq->wqe_cmpl = lpfc_nvmet_sol_fcp_abort_cmp;
+ abts_wqeq->iocb_cmpl = 0;
+ abts_wqeq->iocb_flag |= LPFC_IO_NVME;
+ abts_wqeq->context2 = ctxp;
+ rc = lpfc_sli4_issue_wqe(phba, LPFC_FCP_RING, abts_wqeq);
+ spin_unlock_irqrestore(&phba->hbalock, flags);
+ if (rc == WQE_SUCCESS)
+ return 0;
+
+ lpfc_sli_release_iocbq(phba, abts_wqeq);
+ lpfc_printf_log(phba, KERN_ERR, LOG_NVME,
+ "6166 Failed abts issue_wqe with status x%x "
+ "for oxid x%x.\n",
+ rc, ctxp->oxid);
+ return 1;
+}
+
+
+static int
+lpfc_nvmet_unsol_fcp_issue_abort(struct lpfc_hba *phba,
+ struct lpfc_nvmet_rcv_ctx *ctxp,
+ uint32_t sid, uint16_t xri)
+{
+ struct lpfc_nvmet_tgtport *tgtp;
+ struct lpfc_iocbq *abts_wqeq;
+ unsigned long flags;
+ int rc;
+
+ tgtp = (struct lpfc_nvmet_tgtport *)phba->targetport->private;
+ if (!ctxp->wqeq) {
+ ctxp->wqeq = ctxp->rqb_buffer->iocbq;
+ ctxp->wqeq->hba_wqidx = 0;
+ }
+
+ rc = lpfc_nvmet_unsol_issue_abort(phba, ctxp, sid, xri);
+ if (rc == 0)
+ goto aerr;
+
+ spin_lock_irqsave(&phba->hbalock, flags);
+ abts_wqeq = ctxp->wqeq;
+ abts_wqeq->wqe_cmpl = lpfc_nvmet_xmt_fcp_abort_cmp;
+ abts_wqeq->iocb_cmpl = 0;
+ abts_wqeq->iocb_flag |= LPFC_IO_NVMET;
+ rc = lpfc_sli4_issue_wqe(phba, LPFC_FCP_RING, abts_wqeq);
+ spin_unlock_irqrestore(&phba->hbalock, flags);
+ if (rc == WQE_SUCCESS) {
+ atomic_inc(&tgtp->xmt_abort_rsp);
+ return 0;
+ }
+
+aerr:
+ lpfc_nvmet_rq_post(phba, ctxp, &ctxp->rqb_buffer->hbuf);
+ atomic_inc(&tgtp->xmt_abort_rsp_error);
+ lpfc_printf_log(phba, KERN_WARNING, LOG_NVME_ABTS,
+ "6135 Failed to Issue ABTS for oxid x%x. Status x%x\n",
+ ctxp->oxid, rc);
+ return 1;
+}
+
+static int
+lpfc_nvmet_unsol_ls_issue_abort(struct lpfc_hba *phba,
+ struct lpfc_nvmet_rcv_ctx *ctxp,
+ uint32_t sid, uint16_t xri)
+{
+ struct lpfc_nvmet_tgtport *tgtp;
+ struct lpfc_iocbq *abts_wqeq;
+ union lpfc_wqe *wqe_abts;
+ unsigned long flags;
+ int rc;
+
+ tgtp = (struct lpfc_nvmet_tgtport *)phba->targetport->private;
+ if (!ctxp->wqeq) {
+ /* Issue ABTS for this WQE based on iotag */
+ ctxp->wqeq = lpfc_sli_get_iocbq(phba);
+ if (!ctxp->wqeq) {
+ lpfc_printf_log(phba, KERN_WARNING, LOG_NVME_ABTS,
+ "6068 Abort failed: No wqeqs: "
+ "xri: x%x\n", xri);
+ /* No failure to an ABTS request. */
+ kfree(ctxp);
+ return 0;
+ }
+ }
+ abts_wqeq = ctxp->wqeq;
+ wqe_abts = &abts_wqeq->wqe;
+ lpfc_nvmet_unsol_issue_abort(phba, ctxp, sid, xri);
+
+ spin_lock_irqsave(&phba->hbalock, flags);
+ abts_wqeq->wqe_cmpl = lpfc_nvmet_xmt_ls_abort_cmp;
+ abts_wqeq->iocb_cmpl = 0;
+ abts_wqeq->iocb_flag |= LPFC_IO_NVME_LS;
+ rc = lpfc_sli4_issue_wqe(phba, LPFC_ELS_RING, abts_wqeq);
+ spin_unlock_irqrestore(&phba->hbalock, flags);
+ if (rc == WQE_SUCCESS) {
+ atomic_inc(&tgtp->xmt_abort_rsp);
+ return 0;
+ }
+
+ atomic_inc(&tgtp->xmt_abort_rsp_error);
+ abts_wqeq->context2 = NULL;
+ abts_wqeq->context3 = NULL;
+ lpfc_sli_release_iocbq(phba, abts_wqeq);
+ kfree(ctxp);
+ lpfc_printf_log(phba, KERN_WARNING, LOG_NVME_ABTS,
+ "6056 Failed to Issue ABTS. Status x%x\n", rc);
+ return 0;
+}
--- /dev/null
+/*******************************************************************
+ * This file is part of the Emulex Linux Device Driver for *
+ * Fibre Channel Host Bus Adapters. *
+ * Copyright (C) 2017 Broadcom. All Rights Reserved. The term *
+ * “Broadcom” refers to Broadcom Limited and/or its subsidiaries. *
+ * Copyright (C) 2004-2016 Emulex. All rights reserved. *
+ * EMULEX and SLI are trademarks of Emulex. *
+ * www.broadcom.com *
+ * Portions Copyright (C) 2004-2005 Christoph Hellwig *
+ * *
+ * This program is free software; you can redistribute it and/or *
+ * modify it under the terms of version 2 of the GNU General *
+ * Public License as published by the Free Software Foundation. *
+ * This program is distributed in the hope that it will be useful. *
+ * ALL EXPRESS OR IMPLIED CONDITIONS, REPRESENTATIONS AND *
+ * WARRANTIES, INCLUDING ANY IMPLIED WARRANTY OF MERCHANTABILITY, *
+ * FITNESS FOR A PARTICULAR PURPOSE, OR NON-INFRINGEMENT, ARE *
+ * DISCLAIMED, EXCEPT TO THE EXTENT THAT SUCH DISCLAIMERS ARE HELD *
+ * TO BE LEGALLY INVALID. See the GNU General Public License for *
+ * more details, a copy of which can be found in the file COPYING *
+ * included with this package. *
+ ********************************************************************/
+
+#define LPFC_NVMET_MIN_SEGS 16
+#define LPFC_NVMET_DEFAULT_SEGS 64 /* 256K IOs */
+#define LPFC_NVMET_MAX_SEGS 510
+#define LPFC_NVMET_SUCCESS_LEN 12
+
+/* Used for NVME Target */
+struct lpfc_nvmet_tgtport {
+ struct lpfc_hba *phba;
+ struct completion tport_unreg_done;
+
+ /* Stats counters - lpfc_nvmet_unsol_ls_buffer */
+ atomic_t rcv_ls_req_in;
+ atomic_t rcv_ls_req_out;
+ atomic_t rcv_ls_req_drop;
+ atomic_t xmt_ls_abort;
+
+ /* Stats counters - lpfc_nvmet_xmt_ls_rsp */
+ atomic_t xmt_ls_rsp;
+ atomic_t xmt_ls_drop;
+
+ /* Stats counters - lpfc_nvmet_xmt_ls_rsp_cmp */
+ atomic_t xmt_ls_rsp_error;
+ atomic_t xmt_ls_rsp_cmpl;
+
+ /* Stats counters - lpfc_nvmet_unsol_fcp_buffer */
+ atomic_t rcv_fcp_cmd_in;
+ atomic_t rcv_fcp_cmd_out;
+ atomic_t rcv_fcp_cmd_drop;
+
+ /* Stats counters - lpfc_nvmet_xmt_fcp_op */
+ atomic_t xmt_fcp_abort;
+ atomic_t xmt_fcp_drop;
+ atomic_t xmt_fcp_read_rsp;
+ atomic_t xmt_fcp_read;
+ atomic_t xmt_fcp_write;
+ atomic_t xmt_fcp_rsp;
+
+ /* Stats counters - lpfc_nvmet_xmt_fcp_op_cmp */
+ atomic_t xmt_fcp_rsp_cmpl;
+ atomic_t xmt_fcp_rsp_error;
+ atomic_t xmt_fcp_rsp_drop;
+
+
+ /* Stats counters - lpfc_nvmet_unsol_issue_abort */
+ atomic_t xmt_abort_rsp;
+ atomic_t xmt_abort_rsp_error;
+
+ /* Stats counters - lpfc_nvmet_xmt_abort_cmp */
+ atomic_t xmt_abort_cmpl;
+};
+
+struct lpfc_nvmet_rcv_ctx {
+ union {
+ struct nvmefc_tgt_ls_req ls_req;
+ struct nvmefc_tgt_fcp_req fcp_req;
+ } ctx;
+ struct lpfc_hba *phba;
+ struct lpfc_iocbq *wqeq;
+ struct lpfc_iocbq *abort_wqeq;
+ dma_addr_t txrdy_phys;
+ uint32_t *txrdy;
+ uint32_t sid;
+ uint32_t offset;
+ uint16_t oxid;
+ uint16_t size;
+ uint16_t entry_cnt;
+ uint16_t cpu;
+ uint16_t state;
+ /* States */
+#define LPFC_NVMET_STE_FREE 0
+#define LPFC_NVMET_STE_RCV 1
+#define LPFC_NVMET_STE_DATA 2
+#define LPFC_NVMET_STE_ABORT 3
+#define LPFC_NVMET_STE_RSP 4
+#define LPFC_NVMET_STE_DONE 5
+ uint16_t flag;
+#define LPFC_NVMET_IO_INP 1
+#define LPFC_NVMET_ABORT_OP 2
+ struct rqb_dmabuf *rqb_buffer;
+
+#ifdef CONFIG_SCSI_LPFC_DEBUG_FS
+ uint64_t ts_isr_cmd;
+ uint64_t ts_cmd_nvme;
+ uint64_t ts_nvme_data;
+ uint64_t ts_data_wqput;
+ uint64_t ts_isr_data;
+ uint64_t ts_data_nvme;
+ uint64_t ts_nvme_status;
+ uint64_t ts_status_wqput;
+ uint64_t ts_isr_status;
+ uint64_t ts_status_nvme;
+#endif
+};
/*******************************************************************
* This file is part of the Emulex Linux Device Driver for *
* Fibre Channel Host Bus Adapters. *
+ * Copyright (C) 2017 Broadcom. All Rights Reserved. The term *
+ * “Broadcom” refers to Broadcom Limited and/or its subsidiaries. *
* Copyright (C) 2004-2016 Emulex. All rights reserved. *
* EMULEX and SLI are trademarks of Emulex. *
- * www.emulex.com *
+ * www.broadcom.com *
* Portions Copyright (C) 2004-2005 Christoph Hellwig *
* *
* This program is free software; you can redistribute it and/or *
* struct fcp_cmnd, struct fcp_rsp and the number of bde's
* necessary to support the sg_tablesize.
*/
- psb->data = pci_pool_zalloc(phba->lpfc_scsi_dma_buf_pool,
+ psb->data = pci_pool_zalloc(phba->lpfc_sg_dma_buf_pool,
GFP_KERNEL, &psb->dma_handle);
if (!psb->data) {
kfree(psb);
/* Allocate iotag for psb->cur_iocbq. */
iotag = lpfc_sli_next_iotag(phba, &psb->cur_iocbq);
if (iotag == 0) {
- pci_pool_free(phba->lpfc_scsi_dma_buf_pool,
- psb->data, psb->dma_handle);
+ pci_pool_free(phba->lpfc_sg_dma_buf_pool,
+ psb->data, psb->dma_handle);
kfree(psb);
break;
}
struct lpfc_scsi_buf *psb, *next_psb;
unsigned long iflag = 0;
+ if (!(phba->cfg_enable_fc4_type & LPFC_ENABLE_FCP))
+ return;
spin_lock_irqsave(&phba->hbalock, iflag);
spin_lock(&phba->sli4_hba.abts_scsi_buf_list_lock);
list_for_each_entry_safe(psb, next_psb,
int i;
struct lpfc_nodelist *ndlp;
int rrq_empty = 0;
- struct lpfc_sli_ring *pring = &phba->sli.ring[LPFC_ELS_RING];
+ struct lpfc_sli_ring *pring = phba->sli4_hba.els_wq->pring;
+ if (!(phba->cfg_enable_fc4_type & LPFC_ENABLE_FCP))
+ return;
spin_lock_irqsave(&phba->hbalock, iflag);
spin_lock(&phba->sli4_hba.abts_scsi_buf_list_lock);
list_for_each_entry_safe(psb, next_psb,
* for the struct fcp_cmnd, struct fcp_rsp and the number
* of bde's necessary to support the sg_tablesize.
*/
- psb->data = pci_pool_zalloc(phba->lpfc_scsi_dma_buf_pool,
+ psb->data = pci_pool_zalloc(phba->lpfc_sg_dma_buf_pool,
GFP_KERNEL, &psb->dma_handle);
if (!psb->data) {
kfree(psb);
*/
if (phba->cfg_enable_bg && (((unsigned long)(psb->data) &
(unsigned long)(SLI4_PAGE_SIZE - 1)) != 0)) {
- pci_pool_free(phba->lpfc_scsi_dma_buf_pool,
+ pci_pool_free(phba->lpfc_sg_dma_buf_pool,
psb->data, psb->dma_handle);
kfree(psb);
break;
lxri = lpfc_sli4_next_xritag(phba);
if (lxri == NO_XRI) {
- pci_pool_free(phba->lpfc_scsi_dma_buf_pool,
- psb->data, psb->dma_handle);
+ pci_pool_free(phba->lpfc_sg_dma_buf_pool,
+ psb->data, psb->dma_handle);
kfree(psb);
break;
}
/* Allocate iotag for psb->cur_iocbq. */
iotag = lpfc_sli_next_iotag(phba, &psb->cur_iocbq);
if (iotag == 0) {
- pci_pool_free(phba->lpfc_scsi_dma_buf_pool,
- psb->data, psb->dma_handle);
+ pci_pool_free(phba->lpfc_sg_dma_buf_pool,
+ psb->data, psb->dma_handle);
kfree(psb);
lpfc_printf_log(phba, KERN_ERR, LOG_FCP,
"3368 Failed to allocate IOTAG for"
phba->sli4_hba.scsi_xri_cnt++;
spin_unlock_irq(&phba->scsi_buf_list_get_lock);
}
- lpfc_printf_log(phba, KERN_INFO, LOG_BG,
+ lpfc_printf_log(phba, KERN_INFO, LOG_BG | LOG_FCP,
"3021 Allocate %d out of %d requested new SCSI "
"buffers\n", bcnt, num_to_alloc);
}
}
chann = atomic_add_return(1, &phba->fcp_qidx);
- chann = (chann % phba->cfg_fcp_io_channel);
+ chann = chann % phba->cfg_fcp_io_channel;
return chann;
}
struct Scsi_Host *shost;
uint32_t logit = LOG_FCP;
+ phba->fc4ScsiIoCmpls++;
+
/* Sanity check on return of outstanding command */
cmd = lpfc_cmd->pCmd;
if (!cmd)
lpfc_cmd->prot_data_segment = NULL;
}
#endif
+
if (pnode && NLP_CHK_NODE_ACT(pnode))
atomic_dec(&pnode->cmd_pending);
vport->cfg_first_burst_size;
}
fcp_cmnd->fcpCntl3 = WRITE_DATA;
- phba->fc4OutputRequests++;
+ phba->fc4ScsiOutputRequests++;
} else {
iocb_cmd->ulpCommand = CMD_FCP_IREAD64_CR;
iocb_cmd->ulpPU = PARM_READ_CHECK;
fcp_cmnd->fcpCntl3 = READ_DATA;
- phba->fc4InputRequests++;
+ phba->fc4ScsiInputRequests++;
}
} else {
iocb_cmd->ulpCommand = CMD_FCP_ICMND64_CR;
iocb_cmd->un.fcpi.fcpi_parm = 0;
iocb_cmd->ulpPU = 0;
fcp_cmnd->fcpCntl3 = 0;
- phba->fc4ControlRequests++;
+ phba->fc4ScsiControlRequests++;
}
if (phba->sli_rev == 3 &&
!(phba->sli3_options & LPFC_SLI3_BG_ENABLED))
unsigned long poll_tmo_expires =
(jiffies + msecs_to_jiffies(phba->cfg_poll_tmo));
- if (!list_empty(&phba->sli.ring[LPFC_FCP_RING].txcmplq))
+ if (!list_empty(&phba->sli.sli3_ring[LPFC_FCP_RING].txcmplq))
mod_timer(&phba->fcp_poll_timer,
poll_tmo_expires);
}
if (phba->cfg_poll & ENABLE_FCP_RING_POLLING) {
lpfc_sli_handle_fast_ring_event(phba,
- &phba->sli.ring[LPFC_FCP_RING], HA_R0RE_REQ);
+ &phba->sli.sli3_ring[LPFC_FCP_RING], HA_R0RE_REQ);
if (phba->cfg_poll & DISABLE_FCP_RING_INT)
lpfc_poll_rearm_timer(phba);
if (lpfc_cmd == NULL) {
lpfc_rampdown_queue_depth(phba);
- lpfc_printf_vlog(vport, KERN_INFO, LOG_MISC,
+ lpfc_printf_vlog(vport, KERN_INFO, LOG_FCP_ERROR,
"0707 driver's buffer pool is empty, "
"IO busied\n");
goto out_host_busy;
}
if (phba->cfg_poll & ENABLE_FCP_RING_POLLING) {
lpfc_sli_handle_fast_ring_event(phba,
- &phba->sli.ring[LPFC_FCP_RING], HA_R0RE_REQ);
+ &phba->sli.sli3_ring[LPFC_FCP_RING], HA_R0RE_REQ);
if (phba->cfg_poll & DISABLE_FCP_RING_INT)
lpfc_poll_rearm_timer(phba);
IOCB_t *cmd, *icmd;
int ret = SUCCESS, status = 0;
struct lpfc_sli_ring *pring_s4;
- int ring_number, ret_val;
+ int ret_val;
unsigned long flags, iflags;
DECLARE_WAIT_QUEUE_HEAD_ONSTACK(waitq);
icmd->ulpClass = cmd->ulpClass;
/* ABTS WQE must go to the same WQ as the WQE to be aborted */
- abtsiocb->fcp_wqidx = iocb->fcp_wqidx;
+ abtsiocb->hba_wqidx = iocb->hba_wqidx;
abtsiocb->iocb_flag |= LPFC_USE_FCPWQIDX;
if (iocb->iocb_flag & LPFC_IO_FOF)
abtsiocb->iocb_flag |= LPFC_IO_FOF;
abtsiocb->iocb_cmpl = lpfc_sli_abort_fcp_cmpl;
abtsiocb->vport = vport;
if (phba->sli_rev == LPFC_SLI_REV4) {
- ring_number = MAX_SLI3_CONFIGURED_RINGS + iocb->fcp_wqidx;
- pring_s4 = &phba->sli.ring[ring_number];
+ pring_s4 = lpfc_sli4_calc_ring(phba, iocb);
+ if (pring_s4 == NULL) {
+ ret = FAILED;
+ goto out_unlock;
+ }
/* Note: both hbalock and ring_lock must be set here */
spin_lock_irqsave(&pring_s4->ring_lock, iflags);
ret_val = __lpfc_sli_issue_iocb(phba, pring_s4->ringno,
if (phba->cfg_poll & DISABLE_FCP_RING_INT)
lpfc_sli_handle_fast_ring_event(phba,
- &phba->sli.ring[LPFC_FCP_RING], HA_R0RE_REQ);
+ &phba->sli.sli3_ring[LPFC_FCP_RING], HA_R0RE_REQ);
wait_for_cmpl:
lpfc_cmd->waitq = &waitq;
cnt = lpfc_sli_sum_iocb(vport, tgt_id, lun_id, context);
if (cnt)
lpfc_sli_abort_taskmgmt(vport,
- &phba->sli.ring[phba->sli.fcp_ring],
+ &phba->sli.sli3_ring[LPFC_FCP_RING],
tgt_id, lun_id, context);
later = msecs_to_jiffies(2 * vport->cfg_devloss_tmo * 1000) + jiffies;
while (time_after(later, jiffies) && cnt) {
continue;
if (ndlp->nlp_state == NLP_STE_MAPPED_NODE &&
ndlp->nlp_sid == i &&
- ndlp->rport) {
+ ndlp->rport &&
+ ndlp->nlp_type & NLP_FCP_TARGET) {
match = 1;
break;
}
if (phba->cfg_poll & ENABLE_FCP_RING_POLLING) {
lpfc_sli_handle_fast_ring_event(phba,
- &phba->sli.ring[LPFC_FCP_RING], HA_R0RE_REQ);
+ &phba->sli.sli3_ring[LPFC_FCP_RING], HA_R0RE_REQ);
if (phba->cfg_poll & DISABLE_FCP_RING_INT)
lpfc_poll_rearm_timer(phba);
}
return false;
}
+static int
+lpfc_no_command(struct Scsi_Host *shost, struct scsi_cmnd *cmnd)
+{
+ return SCSI_MLQUEUE_HOST_BUSY;
+}
+
+static int
+lpfc_no_handler(struct scsi_cmnd *cmnd)
+{
+ return FAILED;
+}
+
+static int
+lpfc_no_slave(struct scsi_device *sdev)
+{
+ return -ENODEV;
+}
+
+struct scsi_host_template lpfc_template_nvme = {
+ .module = THIS_MODULE,
+ .name = LPFC_DRIVER_NAME,
+ .proc_name = LPFC_DRIVER_NAME,
+ .info = lpfc_info,
+ .queuecommand = lpfc_no_command,
+ .eh_abort_handler = lpfc_no_handler,
+ .eh_device_reset_handler = lpfc_no_handler,
+ .eh_target_reset_handler = lpfc_no_handler,
+ .eh_bus_reset_handler = lpfc_no_handler,
+ .eh_host_reset_handler = lpfc_no_handler,
+ .slave_alloc = lpfc_no_slave,
+ .slave_configure = lpfc_no_slave,
+ .scan_finished = lpfc_scan_finished,
+ .this_id = -1,
+ .sg_tablesize = 1,
+ .cmd_per_lun = 1,
+ .use_clustering = ENABLE_CLUSTERING,
+ .shost_attrs = lpfc_hba_attrs,
+ .max_sectors = 0xFFFF,
+ .vendor_id = LPFC_NL_VENDOR_ID,
+ .track_queue_depth = 0,
+};
+
struct scsi_host_template lpfc_template_s3 = {
.module = THIS_MODULE,
.name = LPFC_DRIVER_NAME,
/*******************************************************************
* This file is part of the Emulex Linux Device Driver for *
* Fibre Channel Host Bus Adapters. *
+ * Copyright (C) 2017 Broadcom. All Rights Reserved. The term *
+ * “Broadcom” refers to Broadcom Limited and/or its subsidiaries. *
* Copyright (C) 2004-2016 Emulex. All rights reserved. *
* EMULEX and SLI are trademarks of Emulex. *
- * www.emulex.com *
+ * www.broadcom.com *
* *
* This program is free software; you can redistribute it and/or *
* modify it under the terms of version 2 of the GNU General *
uint32_t timeout;
+ uint16_t flags; /* TBD convert exch_busy to flags */
+#define LPFC_SBUF_XBUSY 0x1 /* SLI4 hba reported XB on WCQE cmpl */
uint16_t exch_busy; /* SLI4 hba reported XB on complete WCQE */
uint16_t status; /* From IOCB Word 7- ulpStatus */
uint32_t result; /* From IOCB Word 4. */
* Iotag is in here
*/
struct lpfc_iocbq cur_iocbq;
+ uint16_t cpu;
+
wait_queue_head_t *waitq;
unsigned long start_time;
#endif
};
-#define LPFC_SCSI_DMA_EXT_SIZE 264
-#define LPFC_BPL_SIZE 1024
-#define MDAC_DIRECT_CMD 0x22
+#define LPFC_SCSI_DMA_EXT_SIZE 264
+#define LPFC_BPL_SIZE 1024
+#define MDAC_DIRECT_CMD 0x22
+
+#define FIND_FIRST_OAS_LUN 0
+#define NO_MORE_OAS_LUN -1
+#define NOT_OAS_ENABLED_LUN NO_MORE_OAS_LUN
-#define FIND_FIRST_OAS_LUN 0
-#define NO_MORE_OAS_LUN -1
-#define NOT_OAS_ENABLED_LUN NO_MORE_OAS_LUN
+#define TXRDY_PAYLOAD_LEN 12
int lpfc_sli4_scmd_to_wqidx_distr(struct lpfc_hba *phba,
struct lpfc_scsi_buf *lpfc_cmd);
/*******************************************************************
* This file is part of the Emulex Linux Device Driver for *
* Fibre Channel Host Bus Adapters. *
+ * Copyright (C) 2017 Broadcom. All Rights Reserved. The term *
+ * “Broadcom” refers to Broadcom Limited and/or its subsidiaries. *
* Copyright (C) 2004-2016 Emulex. All rights reserved. *
* EMULEX and SLI are trademarks of Emulex. *
- * www.emulex.com *
+ * www.broadcom.com *
* Portions Copyright (C) 2004-2005 Christoph Hellwig *
* *
* This program is free software; you can redistribute it and/or *
#include <scsi/fc/fc_fs.h>
#include <linux/aer.h>
+#include <linux/nvme-fc-driver.h>
+
#include "lpfc_hw4.h"
#include "lpfc_hw.h"
#include "lpfc_sli.h"
#include "lpfc_sli4.h"
#include "lpfc_nl.h"
#include "lpfc_disc.h"
-#include "lpfc_scsi.h"
#include "lpfc.h"
+#include "lpfc_scsi.h"
+#include "lpfc_nvme.h"
+#include "lpfc_nvmet.h"
#include "lpfc_crtn.h"
#include "lpfc_logmsg.h"
#include "lpfc_compat.h"
struct lpfc_iocbq *);
static void lpfc_sli4_send_seq_to_ulp(struct lpfc_vport *,
struct hbq_dmabuf *);
-static int lpfc_sli4_fp_handle_wcqe(struct lpfc_hba *, struct lpfc_queue *,
+static int lpfc_sli4_fp_handle_cqe(struct lpfc_hba *, struct lpfc_queue *,
struct lpfc_cqe *);
-static int lpfc_sli4_post_els_sgl_list(struct lpfc_hba *, struct list_head *,
+static int lpfc_sli4_post_sgl_list(struct lpfc_hba *, struct list_head *,
int);
static void lpfc_sli4_hba_handle_eqe(struct lpfc_hba *, struct lpfc_eqe *,
uint32_t);
static bool lpfc_sli4_mbox_completions_pending(struct lpfc_hba *phba);
static bool lpfc_sli4_process_missed_mbox_completions(struct lpfc_hba *phba);
+static int lpfc_sli4_abort_nvme_io(struct lpfc_hba *phba,
+ struct lpfc_sli_ring *pring,
+ struct lpfc_iocbq *cmdiocb);
static IOCB_t *
lpfc_get_iocb_from_iocbq(struct lpfc_iocbq *iocbq)
/*
* insert barrier for instruction interlock : data from the hardware
* must have the valid bit checked before it can be copied and acted
- * upon. Given what was seen in lpfc_sli4_cq_get() of speculative
- * instructions allowing action on content before valid bit checked,
- * add barrier here as well. May not be needed as "content" is a
- * single 32-bit entity here (vs multi word structure for cq's).
+ * upon. Speculative instructions were allowing a bcopy at the start
+ * of lpfc_sli4_fp_handle_wcqe(), which is called immediately
+ * after our return, to copy data before the valid bit check above
+ * was done. As such, some of the copied data was stale. The barrier
+ * ensures the check is before any data is copied.
*/
mb();
return eqe;
/*
* insert barrier for instruction interlock : data from the hardware
* must have the valid bit checked before it can be copied and acted
- * upon. Speculative instructions were allowing a bcopy at the start
- * of lpfc_sli4_fp_handle_wcqe(), which is called immediately
- * after our return, to copy data before the valid bit check above
- * was done. As such, some of the copied data was stale. The barrier
- * ensures the check is before any data is copied.
+ * upon. Given what was seen in lpfc_sli4_cq_get() of speculative
+ * instructions allowing action on content before valid bit checked,
+ * add barrier here as well. May not be needed as "content" is a
+ * single 32-bit entity here (vs multi word structure for cq's).
*/
mb();
return cqe;
* on @q then this function will return -ENOMEM.
* The caller is expected to hold the hbalock when calling this routine.
**/
-static int
+int
lpfc_sli4_rq_put(struct lpfc_queue *hq, struct lpfc_queue *dq,
struct lpfc_rqe *hrqe, struct lpfc_rqe *drqe)
{
*
* Returns sglq ponter = success, NULL = Failure.
**/
-static struct lpfc_sglq *
+struct lpfc_sglq *
__lpfc_clear_active_sglq(struct lpfc_hba *phba, uint16_t xritag)
{
struct lpfc_sglq *sglq;
}
/**
- * __lpfc_sli_get_sglq - Allocates an iocb object from sgl pool
+ * __lpfc_sli_get_els_sglq - Allocates an iocb object from sgl pool
* @phba: Pointer to HBA context object.
* @piocb: Pointer to the iocbq.
*
* allocated sglq object else it returns NULL.
**/
static struct lpfc_sglq *
-__lpfc_sli_get_sglq(struct lpfc_hba *phba, struct lpfc_iocbq *piocbq)
+__lpfc_sli_get_els_sglq(struct lpfc_hba *phba, struct lpfc_iocbq *piocbq)
{
- struct list_head *lpfc_sgl_list = &phba->sli4_hba.lpfc_sgl_list;
+ struct list_head *lpfc_els_sgl_list = &phba->sli4_hba.lpfc_els_sgl_list;
struct lpfc_sglq *sglq = NULL;
struct lpfc_sglq *start_sglq = NULL;
struct lpfc_scsi_buf *lpfc_cmd;
ndlp = piocbq->context1;
}
- list_remove_head(lpfc_sgl_list, sglq, struct lpfc_sglq, list);
+ spin_lock(&phba->sli4_hba.sgl_list_lock);
+ list_remove_head(lpfc_els_sgl_list, sglq, struct lpfc_sglq, list);
start_sglq = sglq;
while (!found) {
if (!sglq)
return NULL;
- if (lpfc_test_rrq_active(phba, ndlp, sglq->sli4_lxritag)) {
+ if (ndlp && ndlp->active_rrqs_xri_bitmap &&
+ test_bit(sglq->sli4_lxritag,
+ ndlp->active_rrqs_xri_bitmap)) {
/* This xri has an rrq outstanding for this DID.
* put it back in the list and get another xri.
*/
- list_add_tail(&sglq->list, lpfc_sgl_list);
+ list_add_tail(&sglq->list, lpfc_els_sgl_list);
sglq = NULL;
- list_remove_head(lpfc_sgl_list, sglq,
+ list_remove_head(lpfc_els_sgl_list, sglq,
struct lpfc_sglq, list);
if (sglq == start_sglq) {
sglq = NULL;
phba->sli4_hba.lpfc_sglq_active_list[sglq->sli4_lxritag] = sglq;
sglq->state = SGL_ALLOCATED;
}
+ spin_unlock(&phba->sli4_hba.sgl_list_lock);
+ return sglq;
+}
+
+/**
+ * __lpfc_sli_get_nvmet_sglq - Allocates an iocb object from sgl pool
+ * @phba: Pointer to HBA context object.
+ * @piocb: Pointer to the iocbq.
+ *
+ * This function is called with the sgl_list lock held. This function
+ * gets a new driver sglq object from the sglq list. If the
+ * list is not empty then it is successful, it returns pointer to the newly
+ * allocated sglq object else it returns NULL.
+ **/
+struct lpfc_sglq *
+__lpfc_sli_get_nvmet_sglq(struct lpfc_hba *phba, struct lpfc_iocbq *piocbq)
+{
+ struct list_head *lpfc_nvmet_sgl_list;
+ struct lpfc_sglq *sglq = NULL;
+
+ lpfc_nvmet_sgl_list = &phba->sli4_hba.lpfc_nvmet_sgl_list;
+
+ lockdep_assert_held(&phba->sli4_hba.sgl_list_lock);
+
+ list_remove_head(lpfc_nvmet_sgl_list, sglq, struct lpfc_sglq, list);
+ if (!sglq)
+ return NULL;
+ phba->sli4_hba.lpfc_sglq_active_list[sglq->sli4_lxritag] = sglq;
+ sglq->state = SGL_ALLOCATED;
return sglq;
}
* this IO was aborted then the sglq entry it put on the
* lpfc_abts_els_sgl_list until the CQ_ABORTED_XRI is received. If the
* IO has good status or fails for any other reason then the sglq
- * entry is added to the free list (lpfc_sgl_list).
+ * entry is added to the free list (lpfc_els_sgl_list).
**/
static void
__lpfc_sli_release_iocbq_s4(struct lpfc_hba *phba, struct lpfc_iocbq *iocbq)
struct lpfc_sglq *sglq;
size_t start_clean = offsetof(struct lpfc_iocbq, iocb);
unsigned long iflag = 0;
- struct lpfc_sli_ring *pring = &phba->sli.ring[LPFC_ELS_RING];
+ struct lpfc_sli_ring *pring;
lockdep_assert_held(&phba->hbalock);
if (sglq) {
+ if (iocbq->iocb_flag & LPFC_IO_NVMET) {
+ spin_lock_irqsave(&phba->sli4_hba.sgl_list_lock,
+ iflag);
+ sglq->state = SGL_FREED;
+ sglq->ndlp = NULL;
+ list_add_tail(&sglq->list,
+ &phba->sli4_hba.lpfc_nvmet_sgl_list);
+ spin_unlock_irqrestore(
+ &phba->sli4_hba.sgl_list_lock, iflag);
+ goto out;
+ }
+
+ pring = phba->sli4_hba.els_wq->pring;
if ((iocbq->iocb_flag & LPFC_EXCHANGE_BUSY) &&
(sglq->state != SGL_XRI_ABORTED)) {
- spin_lock_irqsave(&phba->sli4_hba.abts_sgl_list_lock,
- iflag);
+ spin_lock_irqsave(&phba->sli4_hba.sgl_list_lock,
+ iflag);
list_add(&sglq->list,
- &phba->sli4_hba.lpfc_abts_els_sgl_list);
+ &phba->sli4_hba.lpfc_abts_els_sgl_list);
spin_unlock_irqrestore(
- &phba->sli4_hba.abts_sgl_list_lock, iflag);
+ &phba->sli4_hba.sgl_list_lock, iflag);
} else {
- spin_lock_irqsave(&pring->ring_lock, iflag);
+ spin_lock_irqsave(&phba->sli4_hba.sgl_list_lock,
+ iflag);
sglq->state = SGL_FREED;
sglq->ndlp = NULL;
list_add_tail(&sglq->list,
- &phba->sli4_hba.lpfc_sgl_list);
- spin_unlock_irqrestore(&pring->ring_lock, iflag);
+ &phba->sli4_hba.lpfc_els_sgl_list);
+ spin_unlock_irqrestore(
+ &phba->sli4_hba.sgl_list_lock, iflag);
/* Check if TXQ queue needs to be serviced */
if (!list_empty(&pring->txq))
}
}
-
+out:
/*
* Clean all volatile data fields, preserve iotag and node struct.
*/
memset((char *)iocbq + start_clean, 0, sizeof(*iocbq) - start_clean);
iocbq->sli4_lxritag = NO_XRI;
iocbq->sli4_xritag = NO_XRI;
+ iocbq->iocb_flag &= ~(LPFC_IO_NVME | LPFC_IO_NVMET |
+ LPFC_IO_NVME_LS);
list_add_tail(&iocbq->list, &phba->lpfc_iocb_list);
}
if (lpfc_is_link_up(phba) &&
(!list_empty(&pring->txq)) &&
- (pring->ringno != phba->sli.fcp_ring ||
+ (pring->ringno != LPFC_FCP_RING ||
phba->sli.sli_flag & LPFC_PROCESS_LA)) {
while ((iocb = lpfc_sli_next_iocb_slot(phba, pring)) &&
struct hbq_dmabuf *hbq_buf;
unsigned long flags;
int i, hbq_count;
- uint32_t hbqno;
hbq_count = lpfc_sli_hbq_count();
/* Return all memory used by all HBQs */
}
phba->hbqs[i].buffer_count = 0;
}
- /* Return all HBQ buffer that are in-fly */
- list_for_each_entry_safe(dmabuf, next_dmabuf, &phba->rb_pend_list,
- list) {
- hbq_buf = container_of(dmabuf, struct hbq_dmabuf, dbuf);
- list_del(&hbq_buf->dbuf.list);
- if (hbq_buf->tag == -1) {
- (phba->hbqs[LPFC_ELS_HBQ].hbq_free_buffer)
- (phba, hbq_buf);
- } else {
- hbqno = hbq_buf->tag >> 16;
- if (hbqno >= LPFC_MAX_HBQS)
- (phba->hbqs[LPFC_ELS_HBQ].hbq_free_buffer)
- (phba, hbq_buf);
- else
- (phba->hbqs[hbqno].hbq_free_buffer)(phba,
- hbq_buf);
- }
- }
/* Mark the HBQs not in use */
phba->hbq_in_use = 0;
hbqe->bde.addrHigh = le32_to_cpu(putPaddrHigh(physaddr));
hbqe->bde.addrLow = le32_to_cpu(putPaddrLow(physaddr));
- hbqe->bde.tus.f.bdeSize = hbq_buf->size;
+ hbqe->bde.tus.f.bdeSize = hbq_buf->total_size;
hbqe->bde.tus.f.bdeFlags = 0;
hbqe->bde.tus.w = le32_to_cpu(hbqe->bde.tus.w);
hbqe->buffer_tag = le32_to_cpu(hbq_buf->tag);
int rc;
struct lpfc_rqe hrqe;
struct lpfc_rqe drqe;
+ struct lpfc_queue *hrq;
+ struct lpfc_queue *drq;
+
+ if (hbqno != LPFC_ELS_HBQ)
+ return 1;
+ hrq = phba->sli4_hba.hdr_rq;
+ drq = phba->sli4_hba.dat_rq;
lockdep_assert_held(&phba->hbalock);
hrqe.address_lo = putPaddrLow(hbq_buf->hbuf.phys);
hrqe.address_hi = putPaddrHigh(hbq_buf->hbuf.phys);
drqe.address_lo = putPaddrLow(hbq_buf->dbuf.phys);
drqe.address_hi = putPaddrHigh(hbq_buf->dbuf.phys);
- rc = lpfc_sli4_rq_put(phba->sli4_hba.hdr_rq, phba->sli4_hba.dat_rq,
- &hrqe, &drqe);
+ rc = lpfc_sli4_rq_put(hrq, drq, &hrqe, &drqe);
if (rc < 0)
return rc;
- hbq_buf->tag = rc;
+ hbq_buf->tag = (rc | (hbqno << 16));
list_add_tail(&hbq_buf->dbuf.list, &phba->hbqs[hbqno].hbq_buffer_list);
return 0;
}
.add_count = 40,
};
-/* HBQ for the extra ring if needed */
-static struct lpfc_hbq_init lpfc_extra_hbq = {
- .rn = 1,
- .entry_count = 200,
- .mask_count = 0,
- .profile = 0,
- .ring_mask = (1 << LPFC_EXTRA_RING),
- .buffer_count = 0,
- .init_count = 0,
- .add_count = 5,
-};
-
/* Array of HBQs */
struct lpfc_hbq_init *lpfc_hbq_defs[] = {
&lpfc_els_hbq,
- &lpfc_extra_hbq,
};
/**
return container_of(d_buf, struct hbq_dmabuf, dbuf);
}
+/**
+ * lpfc_sli_rqbuf_get - Remove the first dma buffer off of an RQ list
+ * @phba: Pointer to HBA context object.
+ * @hbqno: HBQ number.
+ *
+ * This function removes the first RQ buffer on an RQ buffer list and returns a
+ * pointer to that buffer. If it finds no buffers on the list it returns NULL.
+ **/
+static struct rqb_dmabuf *
+lpfc_sli_rqbuf_get(struct lpfc_hba *phba, struct lpfc_queue *hrq)
+{
+ struct lpfc_dmabuf *h_buf;
+ struct lpfc_rqb *rqbp;
+
+ rqbp = hrq->rqbp;
+ list_remove_head(&rqbp->rqb_buffer_list, h_buf,
+ struct lpfc_dmabuf, list);
+ if (!h_buf)
+ return NULL;
+ rqbp->buffer_count--;
+ return container_of(h_buf, struct rqb_dmabuf, hbuf);
+}
+
/**
* lpfc_sli_hbqbuf_find - Find the hbq buffer associated with a tag
* @phba: Pointer to HBA context object.
{
int i;
+ switch (fch_type) {
+ case FC_TYPE_NVME:
+ lpfc_nvmet_unsol_ls_event(phba, pring, saveq);
+ return 1;
+ default:
+ break;
+ }
+
/* unSolicited Responses */
if (pring->prt[0].profile) {
if (pring->prt[0].lpfc_sli_rcv_unsol_event)
lpfc_sli_iocbq_lookup_by_tag(struct lpfc_hba *phba,
struct lpfc_sli_ring *pring, uint16_t iotag)
{
- struct lpfc_iocbq *cmd_iocb;
+ struct lpfc_iocbq *cmd_iocb = NULL;
lockdep_assert_held(&phba->hbalock);
if (iotag != 0 && iotag <= phba->sli.last_iotag) {
}
lpfc_printf_log(phba, KERN_ERR, LOG_SLI,
- "0372 iotag x%x is out of range: max iotag (x%x)\n",
- iotag, phba->sli.last_iotag);
+ "0372 iotag x%x lookup error: max iotag (x%x) "
+ "iocb_flag x%x\n",
+ iotag, phba->sli.last_iotag,
+ cmd_iocb ? cmd_iocb->iocb_flag : 0xffff);
return NULL;
}
IOERR_SLI_ABORTED);
}
+/**
+ * lpfc_sli_abort_wqe_ring - Abort all iocbs in the ring
+ * @phba: Pointer to HBA context object.
+ * @pring: Pointer to driver SLI ring object.
+ *
+ * This function aborts all iocbs in the given ring and frees all the iocb
+ * objects in txq. This function issues an abort iocb for all the iocb commands
+ * in txcmplq. The iocbs in the txcmplq is not guaranteed to complete before
+ * the return of this function. The caller is not required to hold any locks.
+ **/
+void
+lpfc_sli_abort_wqe_ring(struct lpfc_hba *phba, struct lpfc_sli_ring *pring)
+{
+ LIST_HEAD(completions);
+ struct lpfc_iocbq *iocb, *next_iocb;
+
+ if (pring->ringno == LPFC_ELS_RING)
+ lpfc_fabric_abort_hba(phba);
+
+ spin_lock_irq(&phba->hbalock);
+ /* Next issue ABTS for everything on the txcmplq */
+ list_for_each_entry_safe(iocb, next_iocb, &pring->txcmplq, list)
+ lpfc_sli4_abort_nvme_io(phba, pring, iocb);
+ spin_unlock_irq(&phba->hbalock);
+}
+
+
/**
* lpfc_sli_abort_fcp_rings - Abort all iocbs in all FCP rings
* @phba: Pointer to HBA context object.
/* Look on all the FCP Rings for the iotag */
if (phba->sli_rev >= LPFC_SLI_REV4) {
for (i = 0; i < phba->cfg_fcp_io_channel; i++) {
- pring = &psli->ring[i + MAX_SLI3_CONFIGURED_RINGS];
+ pring = phba->sli4_hba.fcp_wq[i]->pring;
lpfc_sli_abort_iocb_ring(phba, pring);
}
} else {
- pring = &psli->ring[psli->fcp_ring];
+ pring = &psli->sli3_ring[LPFC_FCP_RING];
lpfc_sli_abort_iocb_ring(phba, pring);
}
}
+/**
+ * lpfc_sli_abort_nvme_rings - Abort all wqes in all NVME rings
+ * @phba: Pointer to HBA context object.
+ *
+ * This function aborts all wqes in NVME rings. This function issues an
+ * abort wqe for all the outstanding IO commands in txcmplq. The iocbs in
+ * the txcmplq is not guaranteed to complete before the return of this
+ * function. The caller is not required to hold any locks.
+ **/
+void
+lpfc_sli_abort_nvme_rings(struct lpfc_hba *phba)
+{
+ struct lpfc_sli_ring *pring;
+ uint32_t i;
+
+ if (phba->sli_rev < LPFC_SLI_REV4)
+ return;
+
+ /* Abort all IO on each NVME ring. */
+ for (i = 0; i < phba->cfg_nvme_io_channel; i++) {
+ pring = phba->sli4_hba.nvme_wq[i]->pring;
+ lpfc_sli_abort_wqe_ring(phba, pring);
+ }
+}
+
/**
* lpfc_sli_flush_fcp_rings - flush all iocbs in the fcp ring
/* Look on all the FCP Rings for the iotag */
if (phba->sli_rev >= LPFC_SLI_REV4) {
for (i = 0; i < phba->cfg_fcp_io_channel; i++) {
- pring = &psli->ring[i + MAX_SLI3_CONFIGURED_RINGS];
+ pring = phba->sli4_hba.fcp_wq[i]->pring;
spin_lock_irq(&pring->ring_lock);
/* Retrieve everything on txq */
IOERR_SLI_DOWN);
}
} else {
- pring = &psli->ring[psli->fcp_ring];
+ pring = &psli->sli3_ring[LPFC_FCP_RING];
spin_lock_irq(&phba->hbalock);
/* Retrieve everything on txq */
}
}
+/**
+ * lpfc_sli_flush_nvme_rings - flush all wqes in the nvme rings
+ * @phba: Pointer to HBA context object.
+ *
+ * This function flushes all wqes in the nvme rings and frees all resources
+ * in the txcmplq. This function does not issue abort wqes for the IO
+ * commands in txcmplq, they will just be returned with
+ * IOERR_SLI_DOWN. This function is invoked with EEH when device's PCI
+ * slot has been permanently disabled.
+ **/
+void
+lpfc_sli_flush_nvme_rings(struct lpfc_hba *phba)
+{
+ LIST_HEAD(txcmplq);
+ struct lpfc_sli_ring *pring;
+ uint32_t i;
+
+ if (phba->sli_rev < LPFC_SLI_REV4)
+ return;
+
+ /* Hint to other driver operations that a flush is in progress. */
+ spin_lock_irq(&phba->hbalock);
+ phba->hba_flag |= HBA_NVME_IOQ_FLUSH;
+ spin_unlock_irq(&phba->hbalock);
+
+ /* Cycle through all NVME rings and complete each IO with
+ * a local driver reason code. This is a flush so no
+ * abort exchange to FW.
+ */
+ for (i = 0; i < phba->cfg_nvme_io_channel; i++) {
+ pring = phba->sli4_hba.nvme_wq[i]->pring;
+
+ /* Retrieve everything on the txcmplq */
+ spin_lock_irq(&pring->ring_lock);
+ list_splice_init(&pring->txcmplq, &txcmplq);
+ pring->txcmplq_cnt = 0;
+ spin_unlock_irq(&pring->ring_lock);
+
+ /* Flush the txcmpq &&&PAE */
+ lpfc_sli_cancel_iocbs(phba, &txcmplq,
+ IOSTAT_LOCAL_REJECT,
+ IOERR_SLI_DOWN);
+ }
+}
+
/**
* lpfc_sli_brdready_s3 - Check for sli3 host ready status
* @phba: Pointer to HBA context object.
/* Initialize relevant SLI info */
for (i = 0; i < psli->num_rings; i++) {
- pring = &psli->ring[i];
+ pring = &psli->sli3_ring[i];
pring->flag = 0;
pring->sli.sli3.rspidx = 0;
pring->sli.sli3.next_cmdidx = 0;
lpfc_sli4_rb_setup(struct lpfc_hba *phba)
{
phba->hbq_in_use = 1;
- phba->hbqs[0].entry_count = lpfc_hbq_defs[0]->entry_count;
+ phba->hbqs[LPFC_ELS_HBQ].entry_count =
+ lpfc_hbq_defs[LPFC_ELS_HBQ]->entry_count;
phba->hbq_count = 1;
+ lpfc_sli_hbqbuf_init_hbqs(phba, LPFC_ELS_HBQ);
/* Initially populate or replenish the HBQs */
- lpfc_sli_hbqbuf_init_hbqs(phba, 0);
return 0;
}
static void
lpfc_sli4_arm_cqeq_intr(struct lpfc_hba *phba)
{
- int fcp_eqidx;
+ int qidx;
lpfc_sli4_cq_release(phba->sli4_hba.mbx_cq, LPFC_QUEUE_REARM);
lpfc_sli4_cq_release(phba->sli4_hba.els_cq, LPFC_QUEUE_REARM);
- fcp_eqidx = 0;
- if (phba->sli4_hba.fcp_cq) {
- do {
- lpfc_sli4_cq_release(phba->sli4_hba.fcp_cq[fcp_eqidx],
- LPFC_QUEUE_REARM);
- } while (++fcp_eqidx < phba->cfg_fcp_io_channel);
- }
+ if (phba->sli4_hba.nvmels_cq)
+ lpfc_sli4_cq_release(phba->sli4_hba.nvmels_cq,
+ LPFC_QUEUE_REARM);
+
+ if (phba->sli4_hba.fcp_cq)
+ for (qidx = 0; qidx < phba->cfg_fcp_io_channel; qidx++)
+ lpfc_sli4_cq_release(phba->sli4_hba.fcp_cq[qidx],
+ LPFC_QUEUE_REARM);
+
+ if (phba->sli4_hba.nvme_cq)
+ for (qidx = 0; qidx < phba->cfg_nvme_io_channel; qidx++)
+ lpfc_sli4_cq_release(phba->sli4_hba.nvme_cq[qidx],
+ LPFC_QUEUE_REARM);
if (phba->cfg_fof)
lpfc_sli4_cq_release(phba->sli4_hba.oas_cq, LPFC_QUEUE_REARM);
- if (phba->sli4_hba.hba_eq) {
- for (fcp_eqidx = 0; fcp_eqidx < phba->cfg_fcp_io_channel;
- fcp_eqidx++)
- lpfc_sli4_eq_release(phba->sli4_hba.hba_eq[fcp_eqidx],
- LPFC_QUEUE_REARM);
+ if (phba->sli4_hba.hba_eq)
+ for (qidx = 0; qidx < phba->io_channel_irqs; qidx++)
+ lpfc_sli4_eq_release(phba->sli4_hba.hba_eq[qidx],
+ LPFC_QUEUE_REARM);
+
+ if (phba->nvmet_support) {
+ for (qidx = 0; qidx < phba->cfg_nvmet_mrq; qidx++) {
+ lpfc_sli4_cq_release(
+ phba->sli4_hba.nvmet_cqset[qidx],
+ LPFC_QUEUE_REARM);
+ }
}
if (phba->cfg_fof)
rsrc_blks->rsrc_size = rsrc_size;
list_add_tail(&rsrc_blks->list, ext_blk_list);
rsrc_start = rsrc_id;
- if ((type == LPFC_RSC_TYPE_FCOE_XRI) && (j == 0))
+ if ((type == LPFC_RSC_TYPE_FCOE_XRI) && (j == 0)) {
phba->sli4_hba.scsi_xri_start = rsrc_start +
- lpfc_sli4_get_els_iocb_cnt(phba);
+ lpfc_sli4_get_iocb_cnt(phba);
+ phba->sli4_hba.nvme_xri_start =
+ phba->sli4_hba.scsi_xri_start +
+ phba->sli4_hba.scsi_xri_max;
+ }
while (rsrc_id < (rsrc_start + rsrc_size)) {
ids[j] = rsrc_id;
return rc;
}
+
+
/**
* lpfc_sli4_dealloc_extent - Deallocate an SLI4 resource extent.
* @phba: Pointer to HBA context object.
}
/**
- * lpfc_sli4_repost_els_sgl_list - Repsot the els buffers sgl pages as block
+ * lpfc_sli4_repost_sgl_list - Repsot the buffers sgl pages as block
* @phba: pointer to lpfc hba data structure.
+ * @pring: Pointer to driver SLI ring object.
+ * @sgl_list: linked link of sgl buffers to post
+ * @cnt: number of linked list buffers
*
- * This routine walks the list of els buffers that have been allocated and
+ * This routine walks the list of buffers that have been allocated and
* repost them to the port by using SGL block post. This is needed after a
* pci_function_reset/warm_start or start. It attempts to construct blocks
- * of els buffer sgls which contains contiguous xris and uses the non-embedded
- * SGL block post mailbox commands to post them to the port. For single els
+ * of buffer sgls which contains contiguous xris and uses the non-embedded
+ * SGL block post mailbox commands to post them to the port. For single
* buffer sgl with non-contiguous xri, if any, it shall use embedded SGL post
* mailbox command for posting.
*
* Returns: 0 = success, non-zero failure.
**/
static int
-lpfc_sli4_repost_els_sgl_list(struct lpfc_hba *phba)
+lpfc_sli4_repost_sgl_list(struct lpfc_hba *phba,
+ struct list_head *sgl_list, int cnt)
{
struct lpfc_sglq *sglq_entry = NULL;
struct lpfc_sglq *sglq_entry_next = NULL;
struct lpfc_sglq *sglq_entry_first = NULL;
- int status, total_cnt, post_cnt = 0, num_posted = 0, block_cnt = 0;
+ int status, total_cnt;
+ int post_cnt = 0, num_posted = 0, block_cnt = 0;
int last_xritag = NO_XRI;
- struct lpfc_sli_ring *pring;
LIST_HEAD(prep_sgl_list);
LIST_HEAD(blck_sgl_list);
LIST_HEAD(allc_sgl_list);
LIST_HEAD(post_sgl_list);
LIST_HEAD(free_sgl_list);
- pring = &phba->sli.ring[LPFC_ELS_RING];
spin_lock_irq(&phba->hbalock);
- spin_lock(&pring->ring_lock);
- list_splice_init(&phba->sli4_hba.lpfc_sgl_list, &allc_sgl_list);
- spin_unlock(&pring->ring_lock);
+ spin_lock(&phba->sli4_hba.sgl_list_lock);
+ list_splice_init(sgl_list, &allc_sgl_list);
+ spin_unlock(&phba->sli4_hba.sgl_list_lock);
spin_unlock_irq(&phba->hbalock);
- total_cnt = phba->sli4_hba.els_xri_cnt;
+ total_cnt = cnt;
list_for_each_entry_safe(sglq_entry, sglq_entry_next,
&allc_sgl_list, list) {
list_del_init(&sglq_entry->list);
/* keep track of last sgl's xritag */
last_xritag = sglq_entry->sli4_xritag;
- /* end of repost sgl list condition for els buffers */
- if (num_posted == phba->sli4_hba.els_xri_cnt) {
+ /* end of repost sgl list condition for buffers */
+ if (num_posted == total_cnt) {
if (post_cnt == 0) {
list_splice_init(&prep_sgl_list,
&blck_sgl_list);
/* Failure, put sgl to free list */
lpfc_printf_log(phba, KERN_WARNING,
LOG_SLI,
- "3159 Failed to post els "
+ "3159 Failed to post "
"sgl, xritag:x%x\n",
sglq_entry->sli4_xritag);
list_add_tail(&sglq_entry->list,
if (post_cnt == 0)
continue;
- /* post the els buffer list sgls as a block */
- status = lpfc_sli4_post_els_sgl_list(phba, &blck_sgl_list,
- post_cnt);
+ /* post the buffer list sgls as a block */
+ status = lpfc_sli4_post_sgl_list(phba, &blck_sgl_list,
+ post_cnt);
if (!status) {
/* success, put sgl list to posted sgl list */
struct lpfc_sglq,
list);
lpfc_printf_log(phba, KERN_WARNING, LOG_SLI,
- "3160 Failed to post els sgl-list, "
+ "3160 Failed to post sgl-list, "
"xritag:x%x-x%x\n",
sglq_entry_first->sli4_xritag,
(sglq_entry_first->sli4_xritag +
if (block_cnt == 0)
last_xritag = NO_XRI;
- /* reset els sgl post count for next round of posting */
+ /* reset sgl post count for next round of posting */
post_cnt = 0;
}
- /* update the number of XRIs posted for ELS */
- phba->sli4_hba.els_xri_cnt = total_cnt;
- /* free the els sgls failed to post */
+ /* free the sgls failed to post */
lpfc_free_sgl_list(phba, &free_sgl_list);
- /* push els sgls posted to the availble list */
+ /* push sgls posted to the available list */
if (!list_empty(&post_sgl_list)) {
spin_lock_irq(&phba->hbalock);
- spin_lock(&pring->ring_lock);
- list_splice_init(&post_sgl_list,
- &phba->sli4_hba.lpfc_sgl_list);
- spin_unlock(&pring->ring_lock);
+ spin_lock(&phba->sli4_hba.sgl_list_lock);
+ list_splice_init(&post_sgl_list, sgl_list);
+ spin_unlock(&phba->sli4_hba.sgl_list_lock);
spin_unlock_irq(&phba->hbalock);
} else {
lpfc_printf_log(phba, KERN_ERR, LOG_SLI,
- "3161 Failure to post els sgl to port.\n");
+ "3161 Failure to post sgl to port.\n");
return -EIO;
}
- return 0;
+
+ /* return the number of XRIs actually posted */
+ return total_cnt;
}
void
int
lpfc_sli4_hba_setup(struct lpfc_hba *phba)
{
- int rc;
+ int rc, i;
LPFC_MBOXQ_t *mboxq;
struct lpfc_mqe *mqe;
uint8_t *vpd;
struct Scsi_Host *shost = lpfc_shost_from_vport(phba->pport);
struct lpfc_vport *vport = phba->pport;
struct lpfc_dmabuf *mp;
+ struct lpfc_rqb *rqbp;
/* Perform a PCI function reset to start from clean */
rc = lpfc_pci_function_reset(phba);
fc_host_node_name(shost) = wwn_to_u64(vport->fc_nodename.u.wwn);
fc_host_port_name(shost) = wwn_to_u64(vport->fc_portname.u.wwn);
- /* update host els and scsi xri-sgl sizes and mappings */
- rc = lpfc_sli4_xri_sgl_update(phba);
+ /* Create all the SLI4 queues */
+ rc = lpfc_sli4_queue_create(phba);
+ if (rc) {
+ lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+ "3089 Failed to allocate queues\n");
+ rc = -ENODEV;
+ goto out_free_mbox;
+ }
+ /* Set up all the queues to the device */
+ rc = lpfc_sli4_queue_setup(phba);
+ if (unlikely(rc)) {
+ lpfc_printf_log(phba, KERN_ERR, LOG_MBOX | LOG_SLI,
+ "0381 Error %d during queue setup.\n ", rc);
+ goto out_stop_timers;
+ }
+ /* Initialize the driver internal SLI layer lists. */
+ lpfc_sli4_setup(phba);
+ lpfc_sli4_queue_init(phba);
+
+ /* update host els xri-sgl sizes and mappings */
+ rc = lpfc_sli4_els_sgl_update(phba);
if (unlikely(rc)) {
lpfc_printf_log(phba, KERN_ERR, LOG_MBOX | LOG_SLI,
"1400 Failed to update xri-sgl size and "
"mapping: %d\n", rc);
- goto out_free_mbox;
+ goto out_destroy_queue;
}
/* register the els sgl pool to the port */
- rc = lpfc_sli4_repost_els_sgl_list(phba);
- if (unlikely(rc)) {
+ rc = lpfc_sli4_repost_sgl_list(phba, &phba->sli4_hba.lpfc_els_sgl_list,
+ phba->sli4_hba.els_xri_cnt);
+ if (unlikely(rc < 0)) {
lpfc_printf_log(phba, KERN_ERR, LOG_MBOX | LOG_SLI,
"0582 Error %d during els sgl post "
"operation\n", rc);
rc = -ENODEV;
- goto out_free_mbox;
+ goto out_destroy_queue;
}
+ phba->sli4_hba.els_xri_cnt = rc;
- /* register the allocated scsi sgl pool to the port */
- rc = lpfc_sli4_repost_scsi_sgl_list(phba);
- if (unlikely(rc)) {
- lpfc_printf_log(phba, KERN_ERR, LOG_MBOX | LOG_SLI,
- "0383 Error %d during scsi sgl post "
- "operation\n", rc);
- /* Some Scsi buffers were moved to the abort scsi list */
- /* A pci function reset will repost them */
- rc = -ENODEV;
- goto out_free_mbox;
+ if (phba->nvmet_support) {
+ /* update host nvmet xri-sgl sizes and mappings */
+ rc = lpfc_sli4_nvmet_sgl_update(phba);
+ if (unlikely(rc)) {
+ lpfc_printf_log(phba, KERN_ERR, LOG_MBOX | LOG_SLI,
+ "6308 Failed to update nvmet-sgl size "
+ "and mapping: %d\n", rc);
+ goto out_destroy_queue;
+ }
+
+ /* register the nvmet sgl pool to the port */
+ rc = lpfc_sli4_repost_sgl_list(
+ phba,
+ &phba->sli4_hba.lpfc_nvmet_sgl_list,
+ phba->sli4_hba.nvmet_xri_cnt);
+ if (unlikely(rc < 0)) {
+ lpfc_printf_log(phba, KERN_ERR, LOG_MBOX | LOG_SLI,
+ "3117 Error %d during nvmet "
+ "sgl post\n", rc);
+ rc = -ENODEV;
+ goto out_destroy_queue;
+ }
+ phba->sli4_hba.nvmet_xri_cnt = rc;
+ lpfc_nvmet_create_targetport(phba);
+ } else {
+ /* update host scsi xri-sgl sizes and mappings */
+ rc = lpfc_sli4_scsi_sgl_update(phba);
+ if (unlikely(rc)) {
+ lpfc_printf_log(phba, KERN_ERR, LOG_MBOX | LOG_SLI,
+ "6309 Failed to update scsi-sgl size "
+ "and mapping: %d\n", rc);
+ goto out_destroy_queue;
+ }
+
+ /* update host nvme xri-sgl sizes and mappings */
+ rc = lpfc_sli4_nvme_sgl_update(phba);
+ if (unlikely(rc)) {
+ lpfc_printf_log(phba, KERN_ERR, LOG_MBOX | LOG_SLI,
+ "6082 Failed to update nvme-sgl size "
+ "and mapping: %d\n", rc);
+ goto out_destroy_queue;
+ }
+ }
+
+ if (phba->nvmet_support && phba->cfg_nvmet_mrq) {
+
+ /* Post initial buffers to all RQs created */
+ for (i = 0; i < phba->cfg_nvmet_mrq; i++) {
+ rqbp = phba->sli4_hba.nvmet_mrq_hdr[i]->rqbp;
+ INIT_LIST_HEAD(&rqbp->rqb_buffer_list);
+ rqbp->rqb_alloc_buffer = lpfc_sli4_nvmet_alloc;
+ rqbp->rqb_free_buffer = lpfc_sli4_nvmet_free;
+ rqbp->entry_count = 256;
+ rqbp->buffer_count = 0;
+
+ /* Divide by 4 and round down to multiple of 16 */
+ rc = (phba->cfg_nvmet_mrq_post >> 2) & 0xfff8;
+ phba->sli4_hba.nvmet_mrq_hdr[i]->entry_repost = rc;
+ phba->sli4_hba.nvmet_mrq_data[i]->entry_repost = rc;
+
+ lpfc_post_rq_buffer(
+ phba, phba->sli4_hba.nvmet_mrq_hdr[i],
+ phba->sli4_hba.nvmet_mrq_data[i],
+ phba->cfg_nvmet_mrq_post);
+ }
+ }
+
+ if (phba->cfg_enable_fc4_type & LPFC_ENABLE_FCP) {
+ /* register the allocated scsi sgl pool to the port */
+ rc = lpfc_sli4_repost_scsi_sgl_list(phba);
+ if (unlikely(rc)) {
+ lpfc_printf_log(phba, KERN_ERR, LOG_MBOX | LOG_SLI,
+ "0383 Error %d during scsi sgl post "
+ "operation\n", rc);
+ /* Some Scsi buffers were moved to abort scsi list */
+ /* A pci function reset will repost them */
+ rc = -ENODEV;
+ goto out_destroy_queue;
+ }
+ }
+
+ if ((phba->cfg_enable_fc4_type & LPFC_ENABLE_NVME) &&
+ (phba->nvmet_support == 0)) {
+
+ /* register the allocated nvme sgl pool to the port */
+ rc = lpfc_repost_nvme_sgl_list(phba);
+ if (unlikely(rc)) {
+ lpfc_printf_log(phba, KERN_ERR, LOG_MBOX | LOG_SLI,
+ "6116 Error %d during nvme sgl post "
+ "operation\n", rc);
+ /* Some NVME buffers were moved to abort nvme list */
+ /* A pci function reset will repost them */
+ rc = -ENODEV;
+ goto out_destroy_queue;
+ }
}
/* Post the rpi header region to the device. */
"0393 Error %d during rpi post operation\n",
rc);
rc = -ENODEV;
- goto out_free_mbox;
+ goto out_destroy_queue;
}
lpfc_sli4_node_prep(phba);
- /* Create all the SLI4 queues */
- rc = lpfc_sli4_queue_create(phba);
- if (rc) {
- lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
- "3089 Failed to allocate queues\n");
- rc = -ENODEV;
- goto out_stop_timers;
- }
- /* Set up all the queues to the device */
- rc = lpfc_sli4_queue_setup(phba);
- if (unlikely(rc)) {
- lpfc_printf_log(phba, KERN_ERR, LOG_MBOX | LOG_SLI,
- "0381 Error %d during queue setup.\n ", rc);
- goto out_destroy_queue;
+ if (!(phba->hba_flag & HBA_FCOE_MODE)) {
+ if ((phba->nvmet_support == 0) || (phba->cfg_nvmet_mrq == 1)) {
+ /*
+ * The FC Port needs to register FCFI (index 0)
+ */
+ lpfc_reg_fcfi(phba, mboxq);
+ mboxq->vport = phba->pport;
+ rc = lpfc_sli_issue_mbox(phba, mboxq, MBX_POLL);
+ if (rc != MBX_SUCCESS)
+ goto out_unset_queue;
+ rc = 0;
+ phba->fcf.fcfi = bf_get(lpfc_reg_fcfi_fcfi,
+ &mboxq->u.mqe.un.reg_fcfi);
+ } else {
+ /* We are a NVME Target mode with MRQ > 1 */
+
+ /* First register the FCFI */
+ lpfc_reg_fcfi_mrq(phba, mboxq, 0);
+ mboxq->vport = phba->pport;
+ rc = lpfc_sli_issue_mbox(phba, mboxq, MBX_POLL);
+ if (rc != MBX_SUCCESS)
+ goto out_unset_queue;
+ rc = 0;
+ phba->fcf.fcfi = bf_get(lpfc_reg_fcfi_mrq_fcfi,
+ &mboxq->u.mqe.un.reg_fcfi_mrq);
+
+ /* Next register the MRQs */
+ lpfc_reg_fcfi_mrq(phba, mboxq, 1);
+ mboxq->vport = phba->pport;
+ rc = lpfc_sli_issue_mbox(phba, mboxq, MBX_POLL);
+ if (rc != MBX_SUCCESS)
+ goto out_unset_queue;
+ rc = 0;
+ }
+ /* Check if the port is configured to be disabled */
+ lpfc_sli_read_link_ste(phba);
}
/* Arm the CQs and then EQs on device */
rc = 0;
}
- if (!(phba->hba_flag & HBA_FCOE_MODE)) {
- /*
- * The FC Port needs to register FCFI (index 0)
- */
- lpfc_reg_fcfi(phba, mboxq);
- mboxq->vport = phba->pport;
- rc = lpfc_sli_issue_mbox(phba, mboxq, MBX_POLL);
- if (rc != MBX_SUCCESS)
- goto out_unset_queue;
- rc = 0;
- phba->fcf.fcfi = bf_get(lpfc_reg_fcfi_fcfi,
- &mboxq->u.mqe.un.reg_fcfi);
-
- /* Check if the port is configured to be disabled */
- lpfc_sli_read_link_ste(phba);
- }
-
/*
* The port is ready, set the host's link state to LINK_DOWN
* in preparation for link interrupts.
/* Find the eq associated with the mcq */
if (phba->sli4_hba.hba_eq)
- for (eqidx = 0; eqidx < phba->cfg_fcp_io_channel; eqidx++)
+ for (eqidx = 0; eqidx < phba->io_channel_irqs; eqidx++)
if (phba->sli4_hba.hba_eq[eqidx]->queue_id ==
phba->sli4_hba.mbx_cq->assoc_qid) {
fpeq = phba->sli4_hba.hba_eq[eqidx];
= MAILBOX_HBA_EXT_OFFSET;
/* Copy the mailbox extension data */
- if (pmbox->in_ext_byte_len && pmbox->context2) {
+ if (pmbox->in_ext_byte_len && pmbox->context2)
lpfc_memcpy_to_slim(phba->MBslimaddr +
MAILBOX_HBA_EXT_OFFSET,
pmbox->context2, pmbox->in_ext_byte_len);
- }
- if (mbx->mbxCommand == MBX_CONFIG_PORT) {
+ if (mbx->mbxCommand == MBX_CONFIG_PORT)
/* copy command data into host mbox for cmpl */
- lpfc_sli_pcimem_bcopy(mbx, phba->mbox, MAILBOX_CMD_SIZE);
- }
+ lpfc_sli_pcimem_bcopy(mbx, phba->mbox,
+ MAILBOX_CMD_SIZE);
/* First copy mbox command data to HBA SLIM, skip past first
word */
writel(ldata, to_slim);
readl(to_slim); /* flush */
- if (mbx->mbxCommand == MBX_CONFIG_PORT) {
+ if (mbx->mbxCommand == MBX_CONFIG_PORT)
/* switch over to host mailbox */
psli->sli_flag |= LPFC_SLI_ACTIVE;
- }
}
wmb();
if (psli->sli_flag & LPFC_SLI_ACTIVE) {
/* copy results back to user */
- lpfc_sli_pcimem_bcopy(phba->mbox, mbx, MAILBOX_CMD_SIZE);
+ lpfc_sli_pcimem_bcopy(phba->mbox, mbx,
+ MAILBOX_CMD_SIZE);
/* Copy the mailbox extension data */
if (pmbox->out_ext_byte_len && pmbox->context2) {
lpfc_sli_pcimem_bcopy(phba->mbox_ext,
} else {
/* First copy command data */
lpfc_memcpy_from_slim(mbx, phba->MBslimaddr,
- MAILBOX_CMD_SIZE);
+ MAILBOX_CMD_SIZE);
/* Copy the mailbox extension data */
if (pmbox->out_ext_byte_len && pmbox->context2) {
lpfc_memcpy_from_slim(pmbox->context2,
{
struct lpfc_iocbq *nextiocb;
IOCB_t *iocb;
- struct lpfc_sli_ring *pring = &phba->sli.ring[ring_number];
+ struct lpfc_sli_ring *pring = &phba->sli.sli3_ring[ring_number];
lockdep_assert_held(&phba->hbalock);
* For FCP commands, we must be in a state where we can process link
* attention events.
*/
- } else if (unlikely(pring->ringno == phba->sli.fcp_ring &&
+ } else if (unlikely(pring->ringno == LPFC_FCP_RING &&
!(phba->sli.sli_flag & LPFC_PROCESS_LA))) {
goto iocb_busy;
}
union lpfc_wqe *wqe;
union lpfc_wqe128 wqe128;
struct lpfc_queue *wq;
- struct lpfc_sli_ring *pring = &phba->sli.ring[ring_number];
+ struct lpfc_sli_ring *pring;
- lockdep_assert_held(&phba->hbalock);
+ /* Get the WQ */
+ if ((piocb->iocb_flag & LPFC_IO_FCP) ||
+ (piocb->iocb_flag & LPFC_USE_FCPWQIDX)) {
+ if (!phba->cfg_fof || (!(piocb->iocb_flag & LPFC_IO_OAS)))
+ wq = phba->sli4_hba.fcp_wq[piocb->hba_wqidx];
+ else
+ wq = phba->sli4_hba.oas_wq;
+ } else {
+ wq = phba->sli4_hba.els_wq;
+ }
+
+ /* Get corresponding ring */
+ pring = wq->pring;
/*
* The WQE can be either 64 or 128 bytes,
*/
wqe = (union lpfc_wqe *)&wqe128;
+ lockdep_assert_held(&phba->hbalock);
+
if (piocb->sli4_xritag == NO_XRI) {
if (piocb->iocb.ulpCommand == CMD_ABORT_XRI_CN ||
piocb->iocb.ulpCommand == CMD_CLOSE_XRI_CN)
return IOCB_BUSY;
}
} else {
- sglq = __lpfc_sli_get_sglq(phba, piocb);
+ sglq = __lpfc_sli_get_els_sglq(phba, piocb);
if (!sglq) {
if (!(flag & SLI_IOCB_RET_IOCB)) {
__lpfc_sli_ringtx_put(phba,
}
}
}
- } else if (piocb->iocb_flag & LPFC_IO_FCP) {
+ } else if (piocb->iocb_flag & LPFC_IO_FCP)
/* These IO's already have an XRI and a mapped sgl. */
sglq = NULL;
- } else {
+ else {
/*
* This is a continuation of a commandi,(CX) so this
* sglq is on the active list
if (lpfc_sli4_iocb2wqe(phba, piocb, wqe))
return IOCB_ERROR;
- if ((piocb->iocb_flag & LPFC_IO_FCP) ||
- (piocb->iocb_flag & LPFC_USE_FCPWQIDX)) {
- if (!phba->cfg_fof || (!(piocb->iocb_flag & LPFC_IO_OAS))) {
- wq = phba->sli4_hba.fcp_wq[piocb->fcp_wqidx];
- } else {
- wq = phba->sli4_hba.oas_wq;
- }
- if (lpfc_sli4_wq_put(wq, wqe))
- return IOCB_ERROR;
- } else {
- if (unlikely(!phba->sli4_hba.els_wq))
- return IOCB_ERROR;
- if (lpfc_sli4_wq_put(phba->sli4_hba.els_wq, wqe))
- return IOCB_ERROR;
- }
+ if (lpfc_sli4_wq_put(wq, wqe))
+ return IOCB_ERROR;
lpfc_sli_ringtxcmpl_put(phba, pring, piocb);
return 0;
}
/**
- * lpfc_sli_calc_ring - Calculates which ring to use
+ * lpfc_sli4_calc_ring - Calculates which ring to use
* @phba: Pointer to HBA context object.
- * @ring_number: Initial ring
* @piocb: Pointer to command iocb.
*
- * For SLI4, FCP IO can deferred to one fo many WQs, based on
- * fcp_wqidx, thus we need to calculate the corresponding ring.
+ * For SLI4 only, FCP IO can deferred to one fo many WQs, based on
+ * hba_wqidx, thus we need to calculate the corresponding ring.
* Since ABORTS must go on the same WQ of the command they are
- * aborting, we use command's fcp_wqidx.
+ * aborting, we use command's hba_wqidx.
*/
-static int
-lpfc_sli_calc_ring(struct lpfc_hba *phba, uint32_t ring_number,
- struct lpfc_iocbq *piocb)
+struct lpfc_sli_ring *
+lpfc_sli4_calc_ring(struct lpfc_hba *phba, struct lpfc_iocbq *piocb)
{
- if (phba->sli_rev < LPFC_SLI_REV4)
- return ring_number;
-
- if (piocb->iocb_flag & (LPFC_IO_FCP | LPFC_USE_FCPWQIDX)) {
+ if (piocb->iocb_flag & (LPFC_IO_FCP | LPFC_USE_FCPWQIDX)) {
if (!(phba->cfg_fof) ||
- (!(piocb->iocb_flag & LPFC_IO_FOF))) {
+ (!(piocb->iocb_flag & LPFC_IO_FOF))) {
if (unlikely(!phba->sli4_hba.fcp_wq))
- return LPFC_HBA_ERROR;
+ return NULL;
/*
- * for abort iocb fcp_wqidx should already
+ * for abort iocb hba_wqidx should already
* be setup based on what work queue we used.
*/
if (!(piocb->iocb_flag & LPFC_USE_FCPWQIDX))
- piocb->fcp_wqidx =
+ piocb->hba_wqidx =
lpfc_sli4_scmd_to_wqidx_distr(phba,
piocb->context1);
- ring_number = MAX_SLI3_CONFIGURED_RINGS +
- piocb->fcp_wqidx;
+ return phba->sli4_hba.fcp_wq[piocb->hba_wqidx]->pring;
} else {
if (unlikely(!phba->sli4_hba.oas_wq))
- return LPFC_HBA_ERROR;
- piocb->fcp_wqidx = 0;
- ring_number = LPFC_FCP_OAS_RING;
+ return NULL;
+ piocb->hba_wqidx = 0;
+ return phba->sli4_hba.oas_wq->pring;
}
+ } else {
+ if (unlikely(!phba->sli4_hba.els_wq))
+ return NULL;
+ piocb->hba_wqidx = 0;
+ return phba->sli4_hba.els_wq->pring;
}
- return ring_number;
}
/**
lpfc_sli_issue_iocb(struct lpfc_hba *phba, uint32_t ring_number,
struct lpfc_iocbq *piocb, uint32_t flag)
{
- struct lpfc_fcp_eq_hdl *fcp_eq_hdl;
+ struct lpfc_hba_eq_hdl *hba_eq_hdl;
struct lpfc_sli_ring *pring;
struct lpfc_queue *fpeq;
struct lpfc_eqe *eqe;
int rc, idx;
if (phba->sli_rev == LPFC_SLI_REV4) {
- ring_number = lpfc_sli_calc_ring(phba, ring_number, piocb);
- if (unlikely(ring_number == LPFC_HBA_ERROR))
+ pring = lpfc_sli4_calc_ring(phba, piocb);
+ if (unlikely(pring == NULL))
return IOCB_ERROR;
- idx = piocb->fcp_wqidx;
- pring = &phba->sli.ring[ring_number];
spin_lock_irqsave(&pring->ring_lock, iflags);
rc = __lpfc_sli_issue_iocb(phba, ring_number, piocb, flag);
spin_unlock_irqrestore(&pring->ring_lock, iflags);
if (lpfc_fcp_look_ahead && (piocb->iocb_flag & LPFC_IO_FCP)) {
- fcp_eq_hdl = &phba->sli4_hba.fcp_eq_hdl[idx];
+ idx = piocb->hba_wqidx;
+ hba_eq_hdl = &phba->sli4_hba.hba_eq_hdl[idx];
- if (atomic_dec_and_test(&fcp_eq_hdl->
- fcp_eq_in_use)) {
+ if (atomic_dec_and_test(&hba_eq_hdl->hba_eq_in_use)) {
/* Get associated EQ with this index */
fpeq = phba->sli4_hba.hba_eq[idx];
lpfc_sli4_eq_release(fpeq,
LPFC_QUEUE_REARM);
}
- atomic_inc(&fcp_eq_hdl->fcp_eq_in_use);
+ atomic_inc(&hba_eq_hdl->hba_eq_in_use);
}
} else {
/* For now, SLI2/3 will still use hbalock */
* only when driver needs to support target mode functionality
* or IP over FC functionalities.
*
- * This function is called with no lock held.
+ * This function is called with no lock held. SLI3 only.
**/
static int
lpfc_extra_ring_setup( struct lpfc_hba *phba)
/* Adjust cmd/rsp ring iocb entries more evenly */
/* Take some away from the FCP ring */
- pring = &psli->ring[psli->fcp_ring];
+ pring = &psli->sli3_ring[LPFC_FCP_RING];
pring->sli.sli3.numCiocb -= SLI2_IOCB_CMD_R1XTRA_ENTRIES;
pring->sli.sli3.numRiocb -= SLI2_IOCB_RSP_R1XTRA_ENTRIES;
pring->sli.sli3.numCiocb -= SLI2_IOCB_CMD_R3XTRA_ENTRIES;
pring->sli.sli3.numRiocb -= SLI2_IOCB_RSP_R3XTRA_ENTRIES;
/* and give them to the extra ring */
- pring = &psli->ring[psli->extra_ring];
+ pring = &psli->sli3_ring[LPFC_EXTRA_RING];
pring->sli.sli3.numCiocb += SLI2_IOCB_CMD_R1XTRA_ENTRIES;
pring->sli.sli3.numRiocb += SLI2_IOCB_RSP_R1XTRA_ENTRIES;
/**
- * lpfc_sli_setup - SLI ring setup function
+ * lpfc_sli4_setup - SLI ring setup function
* @phba: Pointer to HBA context object.
*
* lpfc_sli_setup sets up rings of the SLI interface with
* This function always returns 0.
**/
int
+lpfc_sli4_setup(struct lpfc_hba *phba)
+{
+ struct lpfc_sli_ring *pring;
+
+ pring = phba->sli4_hba.els_wq->pring;
+ pring->num_mask = LPFC_MAX_RING_MASK;
+ pring->prt[0].profile = 0; /* Mask 0 */
+ pring->prt[0].rctl = FC_RCTL_ELS_REQ;
+ pring->prt[0].type = FC_TYPE_ELS;
+ pring->prt[0].lpfc_sli_rcv_unsol_event =
+ lpfc_els_unsol_event;
+ pring->prt[1].profile = 0; /* Mask 1 */
+ pring->prt[1].rctl = FC_RCTL_ELS_REP;
+ pring->prt[1].type = FC_TYPE_ELS;
+ pring->prt[1].lpfc_sli_rcv_unsol_event =
+ lpfc_els_unsol_event;
+ pring->prt[2].profile = 0; /* Mask 2 */
+ /* NameServer Inquiry */
+ pring->prt[2].rctl = FC_RCTL_DD_UNSOL_CTL;
+ /* NameServer */
+ pring->prt[2].type = FC_TYPE_CT;
+ pring->prt[2].lpfc_sli_rcv_unsol_event =
+ lpfc_ct_unsol_event;
+ pring->prt[3].profile = 0; /* Mask 3 */
+ /* NameServer response */
+ pring->prt[3].rctl = FC_RCTL_DD_SOL_CTL;
+ /* NameServer */
+ pring->prt[3].type = FC_TYPE_CT;
+ pring->prt[3].lpfc_sli_rcv_unsol_event =
+ lpfc_ct_unsol_event;
+ return 0;
+}
+
+/**
+ * lpfc_sli_setup - SLI ring setup function
+ * @phba: Pointer to HBA context object.
+ *
+ * lpfc_sli_setup sets up rings of the SLI interface with
+ * number of iocbs per ring and iotags. This function is
+ * called while driver attach to the HBA and before the
+ * interrupts are enabled. So there is no need for locking.
+ *
+ * This function always returns 0. SLI3 only.
+ **/
+int
lpfc_sli_setup(struct lpfc_hba *phba)
{
int i, totiocbsize = 0;
struct lpfc_sli_ring *pring;
psli->num_rings = MAX_SLI3_CONFIGURED_RINGS;
- if (phba->sli_rev == LPFC_SLI_REV4)
- psli->num_rings += phba->cfg_fcp_io_channel;
psli->sli_flag = 0;
- psli->fcp_ring = LPFC_FCP_RING;
- psli->next_ring = LPFC_FCP_NEXT_RING;
- psli->extra_ring = LPFC_EXTRA_RING;
psli->iocbq_lookup = NULL;
psli->iocbq_lookup_len = 0;
psli->last_iotag = 0;
for (i = 0; i < psli->num_rings; i++) {
- pring = &psli->ring[i];
+ pring = &psli->sli3_ring[i];
switch (i) {
case LPFC_FCP_RING: /* ring 0 - FCP */
/* numCiocb and numRiocb are used in config_port */
}
/**
- * lpfc_sli_queue_setup - Queue initialization function
+ * lpfc_sli4_queue_init - Queue initialization function
* @phba: Pointer to HBA context object.
*
- * lpfc_sli_queue_setup sets up mailbox queues and iocb queues for each
+ * lpfc_sli4_queue_init sets up mailbox queues and iocb queues for each
* ring. This function also initializes ring indices of each ring.
* This function is called during the initialization of the SLI
* interface of an HBA.
* This function is called with no lock held and always returns
* 1.
**/
-int
-lpfc_sli_queue_setup(struct lpfc_hba *phba)
+void
+lpfc_sli4_queue_init(struct lpfc_hba *phba)
+{
+ struct lpfc_sli *psli;
+ struct lpfc_sli_ring *pring;
+ int i;
+
+ psli = &phba->sli;
+ spin_lock_irq(&phba->hbalock);
+ INIT_LIST_HEAD(&psli->mboxq);
+ INIT_LIST_HEAD(&psli->mboxq_cmpl);
+ /* Initialize list headers for txq and txcmplq as double linked lists */
+ for (i = 0; i < phba->cfg_fcp_io_channel; i++) {
+ pring = phba->sli4_hba.fcp_wq[i]->pring;
+ pring->flag = 0;
+ pring->ringno = LPFC_FCP_RING;
+ INIT_LIST_HEAD(&pring->txq);
+ INIT_LIST_HEAD(&pring->txcmplq);
+ INIT_LIST_HEAD(&pring->iocb_continueq);
+ spin_lock_init(&pring->ring_lock);
+ }
+ for (i = 0; i < phba->cfg_nvme_io_channel; i++) {
+ pring = phba->sli4_hba.nvme_wq[i]->pring;
+ pring->flag = 0;
+ pring->ringno = LPFC_FCP_RING;
+ INIT_LIST_HEAD(&pring->txq);
+ INIT_LIST_HEAD(&pring->txcmplq);
+ INIT_LIST_HEAD(&pring->iocb_continueq);
+ spin_lock_init(&pring->ring_lock);
+ }
+ pring = phba->sli4_hba.els_wq->pring;
+ pring->flag = 0;
+ pring->ringno = LPFC_ELS_RING;
+ INIT_LIST_HEAD(&pring->txq);
+ INIT_LIST_HEAD(&pring->txcmplq);
+ INIT_LIST_HEAD(&pring->iocb_continueq);
+ spin_lock_init(&pring->ring_lock);
+
+ if (phba->cfg_nvme_io_channel) {
+ pring = phba->sli4_hba.nvmels_wq->pring;
+ pring->flag = 0;
+ pring->ringno = LPFC_ELS_RING;
+ INIT_LIST_HEAD(&pring->txq);
+ INIT_LIST_HEAD(&pring->txcmplq);
+ INIT_LIST_HEAD(&pring->iocb_continueq);
+ spin_lock_init(&pring->ring_lock);
+ }
+
+ if (phba->cfg_fof) {
+ pring = phba->sli4_hba.oas_wq->pring;
+ pring->flag = 0;
+ pring->ringno = LPFC_FCP_RING;
+ INIT_LIST_HEAD(&pring->txq);
+ INIT_LIST_HEAD(&pring->txcmplq);
+ INIT_LIST_HEAD(&pring->iocb_continueq);
+ spin_lock_init(&pring->ring_lock);
+ }
+
+ spin_unlock_irq(&phba->hbalock);
+}
+
+/**
+ * lpfc_sli_queue_init - Queue initialization function
+ * @phba: Pointer to HBA context object.
+ *
+ * lpfc_sli_queue_init sets up mailbox queues and iocb queues for each
+ * ring. This function also initializes ring indices of each ring.
+ * This function is called during the initialization of the SLI
+ * interface of an HBA.
+ * This function is called with no lock held and always returns
+ * 1.
+ **/
+void
+lpfc_sli_queue_init(struct lpfc_hba *phba)
{
struct lpfc_sli *psli;
struct lpfc_sli_ring *pring;
INIT_LIST_HEAD(&psli->mboxq_cmpl);
/* Initialize list headers for txq and txcmplq as double linked lists */
for (i = 0; i < psli->num_rings; i++) {
- pring = &psli->ring[i];
+ pring = &psli->sli3_ring[i];
pring->ringno = i;
pring->sli.sli3.next_cmdidx = 0;
pring->sli.sli3.local_getidx = 0;
pring->sli.sli3.cmdidx = 0;
- pring->flag = 0;
- INIT_LIST_HEAD(&pring->txq);
- INIT_LIST_HEAD(&pring->txcmplq);
INIT_LIST_HEAD(&pring->iocb_continueq);
INIT_LIST_HEAD(&pring->iocb_continue_saveq);
INIT_LIST_HEAD(&pring->postbufq);
+ pring->flag = 0;
+ INIT_LIST_HEAD(&pring->txq);
+ INIT_LIST_HEAD(&pring->txcmplq);
spin_lock_init(&pring->ring_lock);
}
spin_unlock_irq(&phba->hbalock);
- return 1;
}
/**
LIST_HEAD(completions);
struct lpfc_hba *phba = vport->phba;
struct lpfc_sli *psli = &phba->sli;
+ struct lpfc_queue *qp = NULL;
struct lpfc_sli_ring *pring;
struct lpfc_iocbq *iocb, *next_iocb;
int i;
lpfc_cleanup_discovery_resources(vport);
spin_lock_irqsave(&phba->hbalock, flags);
- for (i = 0; i < psli->num_rings; i++) {
- pring = &psli->ring[i];
- prev_pring_flag = pring->flag;
- /* Only slow rings */
- if (pring->ringno == LPFC_ELS_RING) {
- pring->flag |= LPFC_DEFERRED_RING_EVENT;
- /* Set the lpfc data pending flag */
- set_bit(LPFC_DATA_READY, &phba->data_flags);
- }
- /*
- * Error everything on the txq since these iocbs have not been
- * given to the FW yet.
- */
- list_for_each_entry_safe(iocb, next_iocb, &pring->txq, list) {
- if (iocb->vport != vport)
- continue;
- list_move_tail(&iocb->list, &completions);
- }
- /* Next issue ABTS for everything on the txcmplq */
- list_for_each_entry_safe(iocb, next_iocb, &pring->txcmplq,
- list) {
- if (iocb->vport != vport)
+ /*
+ * Error everything on the txq since these iocbs
+ * have not been given to the FW yet.
+ * Also issue ABTS for everything on the txcmplq
+ */
+ if (phba->sli_rev != LPFC_SLI_REV4) {
+ for (i = 0; i < psli->num_rings; i++) {
+ pring = &psli->sli3_ring[i];
+ prev_pring_flag = pring->flag;
+ /* Only slow rings */
+ if (pring->ringno == LPFC_ELS_RING) {
+ pring->flag |= LPFC_DEFERRED_RING_EVENT;
+ /* Set the lpfc data pending flag */
+ set_bit(LPFC_DATA_READY, &phba->data_flags);
+ }
+ list_for_each_entry_safe(iocb, next_iocb,
+ &pring->txq, list) {
+ if (iocb->vport != vport)
+ continue;
+ list_move_tail(&iocb->list, &completions);
+ }
+ list_for_each_entry_safe(iocb, next_iocb,
+ &pring->txcmplq, list) {
+ if (iocb->vport != vport)
+ continue;
+ lpfc_sli_issue_abort_iotag(phba, pring, iocb);
+ }
+ pring->flag = prev_pring_flag;
+ }
+ } else {
+ list_for_each_entry(qp, &phba->sli4_hba.lpfc_wq_list, wq_list) {
+ pring = qp->pring;
+ if (!pring)
continue;
- lpfc_sli_issue_abort_iotag(phba, pring, iocb);
+ if (pring == phba->sli4_hba.els_wq->pring) {
+ pring->flag |= LPFC_DEFERRED_RING_EVENT;
+ /* Set the lpfc data pending flag */
+ set_bit(LPFC_DATA_READY, &phba->data_flags);
+ }
+ prev_pring_flag = pring->flag;
+ spin_lock_irq(&pring->ring_lock);
+ list_for_each_entry_safe(iocb, next_iocb,
+ &pring->txq, list) {
+ if (iocb->vport != vport)
+ continue;
+ list_move_tail(&iocb->list, &completions);
+ }
+ spin_unlock_irq(&pring->ring_lock);
+ list_for_each_entry_safe(iocb, next_iocb,
+ &pring->txcmplq, list) {
+ if (iocb->vport != vport)
+ continue;
+ lpfc_sli_issue_abort_iotag(phba, pring, iocb);
+ }
+ pring->flag = prev_pring_flag;
}
-
- pring->flag = prev_pring_flag;
}
-
spin_unlock_irqrestore(&phba->hbalock, flags);
/* Cancel all the IOCBs from the completions list */
{
LIST_HEAD(completions);
struct lpfc_sli *psli = &phba->sli;
+ struct lpfc_queue *qp = NULL;
struct lpfc_sli_ring *pring;
struct lpfc_dmabuf *buf_ptr;
unsigned long flags = 0;
lpfc_fabric_abort_hba(phba);
spin_lock_irqsave(&phba->hbalock, flags);
- for (i = 0; i < psli->num_rings; i++) {
- pring = &psli->ring[i];
- /* Only slow rings */
- if (pring->ringno == LPFC_ELS_RING) {
- pring->flag |= LPFC_DEFERRED_RING_EVENT;
- /* Set the lpfc data pending flag */
- set_bit(LPFC_DATA_READY, &phba->data_flags);
- }
- /*
- * Error everything on the txq since these iocbs have not been
- * given to the FW yet.
- */
- list_splice_init(&pring->txq, &completions);
+ /*
+ * Error everything on the txq since these iocbs
+ * have not been given to the FW yet.
+ */
+ if (phba->sli_rev != LPFC_SLI_REV4) {
+ for (i = 0; i < psli->num_rings; i++) {
+ pring = &psli->sli3_ring[i];
+ /* Only slow rings */
+ if (pring->ringno == LPFC_ELS_RING) {
+ pring->flag |= LPFC_DEFERRED_RING_EVENT;
+ /* Set the lpfc data pending flag */
+ set_bit(LPFC_DATA_READY, &phba->data_flags);
+ }
+ list_splice_init(&pring->txq, &completions);
+ }
+ } else {
+ list_for_each_entry(qp, &phba->sli4_hba.lpfc_wq_list, wq_list) {
+ pring = qp->pring;
+ if (!pring)
+ continue;
+ spin_lock_irq(&pring->ring_lock);
+ list_splice_init(&pring->txq, &completions);
+ spin_unlock_irq(&pring->ring_lock);
+ if (pring == phba->sli4_hba.els_wq->pring) {
+ pring->flag |= LPFC_DEFERRED_RING_EVENT;
+ /* Set the lpfc data pending flag */
+ set_bit(LPFC_DATA_READY, &phba->data_flags);
+ }
+ }
}
spin_unlock_irqrestore(&phba->hbalock, flags);
struct lpfc_iocbq *abtsiocbp;
IOCB_t *icmd = NULL;
IOCB_t *iabt = NULL;
- int ring_number;
int retval;
unsigned long iflags;
iabt->ulpClass = icmd->ulpClass;
/* ABTS WQE must go to the same WQ as the WQE to be aborted */
- abtsiocbp->fcp_wqidx = cmdiocb->fcp_wqidx;
+ abtsiocbp->hba_wqidx = cmdiocb->hba_wqidx;
if (cmdiocb->iocb_flag & LPFC_IO_FCP)
abtsiocbp->iocb_flag |= LPFC_USE_FCPWQIDX;
if (cmdiocb->iocb_flag & LPFC_IO_FOF)
abtsiocbp->iotag);
if (phba->sli_rev == LPFC_SLI_REV4) {
- ring_number =
- lpfc_sli_calc_ring(phba, pring->ringno, abtsiocbp);
- if (unlikely(ring_number == LPFC_HBA_ERROR))
+ pring = lpfc_sli4_calc_ring(phba, abtsiocbp);
+ if (unlikely(pring == NULL))
return 0;
- pring = &phba->sli.ring[ring_number];
/* Note: both hbalock and ring_lock need to be set here */
spin_lock_irqsave(&pring->ring_lock, iflags);
retval = __lpfc_sli_issue_iocb(phba, pring->ringno,
return retval;
}
+/**
+ * lpfc_sli4_abort_nvme_io - Issue abort for a command iocb
+ * @phba: Pointer to HBA context object.
+ * @pring: Pointer to driver SLI ring object.
+ * @cmdiocb: Pointer to driver command iocb object.
+ *
+ * This function issues an abort iocb for the provided command iocb down to
+ * the port. Other than the case the outstanding command iocb is an abort
+ * request, this function issues abort out unconditionally. This function is
+ * called with hbalock held. The function returns 0 when it fails due to
+ * memory allocation failure or when the command iocb is an abort request.
+ **/
+static int
+lpfc_sli4_abort_nvme_io(struct lpfc_hba *phba, struct lpfc_sli_ring *pring,
+ struct lpfc_iocbq *cmdiocb)
+{
+ struct lpfc_vport *vport = cmdiocb->vport;
+ struct lpfc_iocbq *abtsiocbp;
+ union lpfc_wqe *abts_wqe;
+ int retval;
+
+ /*
+ * There are certain command types we don't want to abort. And we
+ * don't want to abort commands that are already in the process of
+ * being aborted.
+ */
+ if (cmdiocb->iocb.ulpCommand == CMD_ABORT_XRI_CN ||
+ cmdiocb->iocb.ulpCommand == CMD_CLOSE_XRI_CN ||
+ (cmdiocb->iocb_flag & LPFC_DRIVER_ABORTED) != 0)
+ return 0;
+
+ /* issue ABTS for this io based on iotag */
+ abtsiocbp = __lpfc_sli_get_iocbq(phba);
+ if (abtsiocbp == NULL)
+ return 0;
+
+ /* This signals the response to set the correct status
+ * before calling the completion handler
+ */
+ cmdiocb->iocb_flag |= LPFC_DRIVER_ABORTED;
+
+ /* Complete prepping the abort wqe and issue to the FW. */
+ abts_wqe = &abtsiocbp->wqe;
+ bf_set(abort_cmd_ia, &abts_wqe->abort_cmd, 0);
+ bf_set(abort_cmd_criteria, &abts_wqe->abort_cmd, T_XRI_TAG);
+
+ /* Explicitly set reserved fields to zero.*/
+ abts_wqe->abort_cmd.rsrvd4 = 0;
+ abts_wqe->abort_cmd.rsrvd5 = 0;
+
+ /* WQE Common - word 6. Context is XRI tag. Set 0. */
+ bf_set(wqe_xri_tag, &abts_wqe->abort_cmd.wqe_com, 0);
+ bf_set(wqe_ctxt_tag, &abts_wqe->abort_cmd.wqe_com, 0);
+
+ /* word 7 */
+ bf_set(wqe_ct, &abts_wqe->abort_cmd.wqe_com, 0);
+ bf_set(wqe_cmnd, &abts_wqe->abort_cmd.wqe_com, CMD_ABORT_XRI_CX);
+ bf_set(wqe_class, &abts_wqe->abort_cmd.wqe_com,
+ cmdiocb->iocb.ulpClass);
+
+ /* word 8 - tell the FW to abort the IO associated with this
+ * outstanding exchange ID.
+ */
+ abts_wqe->abort_cmd.wqe_com.abort_tag = cmdiocb->sli4_xritag;
+
+ /* word 9 - this is the iotag for the abts_wqe completion. */
+ bf_set(wqe_reqtag, &abts_wqe->abort_cmd.wqe_com,
+ abtsiocbp->iotag);
+
+ /* word 10 */
+ bf_set(wqe_wqid, &abts_wqe->abort_cmd.wqe_com, cmdiocb->hba_wqidx);
+ bf_set(wqe_qosd, &abts_wqe->abort_cmd.wqe_com, 1);
+ bf_set(wqe_lenloc, &abts_wqe->abort_cmd.wqe_com, LPFC_WQE_LENLOC_NONE);
+
+ /* word 11 */
+ bf_set(wqe_cmd_type, &abts_wqe->abort_cmd.wqe_com, OTHER_COMMAND);
+ bf_set(wqe_wqec, &abts_wqe->abort_cmd.wqe_com, 1);
+ bf_set(wqe_cqid, &abts_wqe->abort_cmd.wqe_com, LPFC_WQE_CQ_ID_DEFAULT);
+
+ /* ABTS WQE must go to the same WQ as the WQE to be aborted */
+ abtsiocbp->iocb_flag |= LPFC_IO_NVME;
+ abtsiocbp->vport = vport;
+ abtsiocbp->wqe_cmpl = lpfc_nvme_abort_fcreq_cmpl;
+ retval = lpfc_sli4_issue_wqe(phba, LPFC_FCP_RING, abtsiocbp);
+ if (retval == IOCB_ERROR) {
+ lpfc_printf_vlog(vport, KERN_ERR, LOG_NVME,
+ "6147 Failed abts issue_wqe with status x%x "
+ "for oxid x%x\n",
+ retval, cmdiocb->sli4_xritag);
+ lpfc_sli_release_iocbq(phba, abtsiocbp);
+ return retval;
+ }
+
+ lpfc_printf_vlog(vport, KERN_ERR, LOG_NVME,
+ "6148 Drv Abort NVME Request Issued for "
+ "ox_id x%x on reqtag x%x\n",
+ cmdiocb->sli4_xritag,
+ abtsiocbp->iotag);
+
+ return retval;
+}
+
/**
* lpfc_sli_hba_iocb_abort - Abort all iocbs to an hba.
* @phba: pointer to lpfc HBA data structure.
{
struct lpfc_sli *psli = &phba->sli;
struct lpfc_sli_ring *pring;
+ struct lpfc_queue *qp = NULL;
int i;
- for (i = 0; i < psli->num_rings; i++) {
- pring = &psli->ring[i];
+ if (phba->sli_rev != LPFC_SLI_REV4) {
+ for (i = 0; i < psli->num_rings; i++) {
+ pring = &psli->sli3_ring[i];
+ lpfc_sli_abort_iocb_ring(phba, pring);
+ }
+ return;
+ }
+ list_for_each_entry(qp, &phba->sli4_hba.lpfc_wq_list, wq_list) {
+ pring = qp->pring;
+ if (!pring)
+ continue;
lpfc_sli_abort_iocb_ring(phba, pring);
}
}
abtsiocb->vport = vport;
/* ABTS WQE must go to the same WQ as the WQE to be aborted */
- abtsiocb->fcp_wqidx = iocbq->fcp_wqidx;
+ abtsiocb->hba_wqidx = iocbq->hba_wqidx;
if (iocbq->iocb_flag & LPFC_IO_FCP)
abtsiocb->iocb_flag |= LPFC_USE_FCPWQIDX;
if (iocbq->iocb_flag & LPFC_IO_FOF)
int sum, i, ret_val;
unsigned long iflags;
struct lpfc_sli_ring *pring_s4;
- uint32_t ring_number;
spin_lock_irq(&phba->hbalock);
abtsiocbq->vport = vport;
/* ABTS WQE must go to the same WQ as the WQE to be aborted */
- abtsiocbq->fcp_wqidx = iocbq->fcp_wqidx;
+ abtsiocbq->hba_wqidx = iocbq->hba_wqidx;
if (iocbq->iocb_flag & LPFC_IO_FCP)
abtsiocbq->iocb_flag |= LPFC_USE_FCPWQIDX;
if (iocbq->iocb_flag & LPFC_IO_FOF)
iocbq->iocb_flag |= LPFC_DRIVER_ABORTED;
if (phba->sli_rev == LPFC_SLI_REV4) {
- ring_number = MAX_SLI3_CONFIGURED_RINGS +
- iocbq->fcp_wqidx;
- pring_s4 = &phba->sli.ring[ring_number];
+ pring_s4 = lpfc_sli4_calc_ring(phba, iocbq);
+ if (pring_s4 == NULL)
+ continue;
/* Note: both hbalock and ring_lock must be set here */
spin_lock_irqsave(&pring_s4->ring_lock, iflags);
ret_val = __lpfc_sli_issue_iocb(phba, pring_s4->ringno,
struct lpfc_iocbq *iocb;
int txq_cnt = 0;
int txcmplq_cnt = 0;
- struct lpfc_sli_ring *pring = &phba->sli.ring[LPFC_ELS_RING];
+ struct lpfc_sli_ring *pring;
unsigned long iflags;
bool iocb_completed = true;
+ if (phba->sli_rev >= LPFC_SLI_REV4)
+ pring = lpfc_sli4_calc_ring(phba, piocb);
+ else
+ pring = &phba->sli.sli3_ring[ring_number];
/*
* If the caller has provided a response iocbq buffer, then context2
* is NULL or its an error.
uint32_t ha_copy;
unsigned long status;
unsigned long iflag;
+ struct lpfc_sli_ring *pring;
/* Get the driver's phba structure from the dev_id and
* assume the HBA is not interrupting.
status = (ha_copy & (HA_RXMASK << (4*LPFC_FCP_RING)));
status >>= (4*LPFC_FCP_RING);
+ pring = &phba->sli.sli3_ring[LPFC_FCP_RING];
if (status & HA_RXMASK)
- lpfc_sli_handle_fast_ring_event(phba,
- &phba->sli.ring[LPFC_FCP_RING],
- status);
+ lpfc_sli_handle_fast_ring_event(phba, pring, status);
if (phba->cfg_multi_ring_support == 2) {
/*
status >>= (4*LPFC_EXTRA_RING);
if (status & HA_RXMASK) {
lpfc_sli_handle_fast_ring_event(phba,
- &phba->sli.ring[LPFC_EXTRA_RING],
+ &phba->sli.sli3_ring[LPFC_EXTRA_RING],
status);
}
}
lpfc_sli4_els_wcqe_to_rspiocbq(struct lpfc_hba *phba,
struct lpfc_iocbq *irspiocbq)
{
- struct lpfc_sli_ring *pring = &phba->sli.ring[LPFC_ELS_RING];
+ struct lpfc_sli_ring *pring;
struct lpfc_iocbq *cmdiocbq;
struct lpfc_wcqe_complete *wcqe;
unsigned long iflags;
+ pring = lpfc_phba_elsring(phba);
+
wcqe = &irspiocbq->cq_event.cqe.wcqe_cmpl;
spin_lock_irqsave(&pring->ring_lock, iflags);
pring->stats.iocb_event++;
txq_cnt++;
if (!list_empty(&pring->txcmplq))
txcmplq_cnt++;
- if (!list_empty(&phba->sli.ring[LPFC_FCP_RING].txcmplq))
- fcp_txcmplq_cnt++;
lpfc_printf_log(phba, KERN_ERR, LOG_SLI,
"0387 NO IOCBQ data: txq_cnt=%d iocb_cnt=%d "
"fcp_txcmplq_cnt=%d, els_txcmplq_cnt=%d\n",
lpfc_sli4_sp_handle_rcqe(struct lpfc_hba *phba, struct lpfc_rcqe *rcqe)
{
bool workposted = false;
+ struct fc_frame_header *fc_hdr;
struct lpfc_queue *hrq = phba->sli4_hba.hdr_rq;
struct lpfc_queue *drq = phba->sli4_hba.dat_rq;
struct hbq_dmabuf *dma_buf;
}
hrq->RQ_rcv_buf++;
memcpy(&dma_buf->cq_event.cqe.rcqe_cmpl, rcqe, sizeof(*rcqe));
+
+ /* If a NVME LS event (type 0x28), treat it as Fast path */
+ fc_hdr = (struct fc_frame_header *)dma_buf->hbuf.virt;
+
/* save off the frame for the word thread to process */
list_add_tail(&dma_buf->cq_event.list,
&phba->sli4_hba.sp_queue_event);
return;
}
+ /* Save EQ associated with this CQ */
+ cq->assoc_qp = speq;
+
/* Process all the entries to the CQ */
switch (cq->type) {
case LPFC_MCQ:
break;
case LPFC_WCQ:
while ((cqe = lpfc_sli4_cq_get(cq))) {
- if (cq->subtype == LPFC_FCP)
- workposted |= lpfc_sli4_fp_handle_wcqe(phba, cq,
+ if ((cq->subtype == LPFC_FCP) ||
+ (cq->subtype == LPFC_NVME))
+ workposted |= lpfc_sli4_fp_handle_cqe(phba, cq,
cqe);
else
workposted |= lpfc_sli4_sp_handle_cqe(phba, cq,
bf_get(lpfc_wcqe_c_request_tag, wcqe));
return;
}
- if (unlikely(!cmdiocbq->iocb_cmpl)) {
+
+ if (cq->assoc_qp)
+ cmdiocbq->isr_timestamp =
+ cq->assoc_qp->isr_timestamp;
+
+ if (cmdiocbq->iocb_cmpl == NULL) {
+ if (cmdiocbq->wqe_cmpl) {
+ if (cmdiocbq->iocb_flag & LPFC_DRIVER_ABORTED) {
+ spin_lock_irqsave(&phba->hbalock, iflags);
+ cmdiocbq->iocb_flag &= ~LPFC_DRIVER_ABORTED;
+ spin_unlock_irqrestore(&phba->hbalock, iflags);
+ }
+
+ /* Pass the cmd_iocb and the wcqe to the upper layer */
+ (cmdiocbq->wqe_cmpl)(phba, cmdiocbq, wcqe);
+ return;
+ }
lpfc_printf_log(phba, KERN_WARNING, LOG_SLI,
"0375 FCP cmdiocb not callback function "
"iotag: (%d)\n",
{
struct lpfc_queue *childwq;
bool wqid_matched = false;
- uint16_t fcp_wqid;
+ uint16_t hba_wqid;
/* Check for fast-path FCP work queue release */
- fcp_wqid = bf_get(lpfc_wcqe_r_wq_id, wcqe);
+ hba_wqid = bf_get(lpfc_wcqe_r_wq_id, wcqe);
list_for_each_entry(childwq, &cq->child_list, list) {
- if (childwq->queue_id == fcp_wqid) {
+ if (childwq->queue_id == hba_wqid) {
lpfc_sli4_wq_release(childwq,
bf_get(lpfc_wcqe_r_wqe_index, wcqe));
wqid_matched = true;
if (wqid_matched != true)
lpfc_printf_log(phba, KERN_WARNING, LOG_SLI,
"2580 Fast-path wqe consume event carries "
- "miss-matched qid: wcqe-qid=x%x\n", fcp_wqid);
+ "miss-matched qid: wcqe-qid=x%x\n", hba_wqid);
}
/**
- * lpfc_sli4_fp_handle_wcqe - Process fast-path work queue completion entry
- * @cq: Pointer to the completion queue.
- * @eqe: Pointer to fast-path completion queue entry.
+ * lpfc_sli4_nvmet_handle_rcqe - Process a receive-queue completion queue entry
+ * @phba: Pointer to HBA context object.
+ * @rcqe: Pointer to receive-queue completion queue entry.
*
- * This routine process a fast-path work queue completion entry from fast-path
+ * This routine process a receive-queue completion queue entry.
+ *
+ * Return: true if work posted to worker thread, otherwise false.
+ **/
+static bool
+lpfc_sli4_nvmet_handle_rcqe(struct lpfc_hba *phba, struct lpfc_queue *cq,
+ struct lpfc_rcqe *rcqe)
+{
+ bool workposted = false;
+ struct lpfc_queue *hrq;
+ struct lpfc_queue *drq;
+ struct rqb_dmabuf *dma_buf;
+ struct fc_frame_header *fc_hdr;
+ uint32_t status, rq_id;
+ unsigned long iflags;
+ uint32_t fctl, idx;
+
+ if ((phba->nvmet_support == 0) ||
+ (phba->sli4_hba.nvmet_cqset == NULL))
+ return workposted;
+
+ idx = cq->queue_id - phba->sli4_hba.nvmet_cqset[0]->queue_id;
+ hrq = phba->sli4_hba.nvmet_mrq_hdr[idx];
+ drq = phba->sli4_hba.nvmet_mrq_data[idx];
+
+ /* sanity check on queue memory */
+ if (unlikely(!hrq) || unlikely(!drq))
+ return workposted;
+
+ if (bf_get(lpfc_cqe_code, rcqe) == CQE_CODE_RECEIVE_V1)
+ rq_id = bf_get(lpfc_rcqe_rq_id_v1, rcqe);
+ else
+ rq_id = bf_get(lpfc_rcqe_rq_id, rcqe);
+
+ if ((phba->nvmet_support == 0) ||
+ (rq_id != hrq->queue_id))
+ return workposted;
+
+ status = bf_get(lpfc_rcqe_status, rcqe);
+ switch (status) {
+ case FC_STATUS_RQ_BUF_LEN_EXCEEDED:
+ lpfc_printf_log(phba, KERN_ERR, LOG_SLI,
+ "6126 Receive Frame Truncated!!\n");
+ hrq->RQ_buf_trunc++;
+ break;
+ case FC_STATUS_RQ_SUCCESS:
+ lpfc_sli4_rq_release(hrq, drq);
+ spin_lock_irqsave(&phba->hbalock, iflags);
+ dma_buf = lpfc_sli_rqbuf_get(phba, hrq);
+ if (!dma_buf) {
+ hrq->RQ_no_buf_found++;
+ spin_unlock_irqrestore(&phba->hbalock, iflags);
+ goto out;
+ }
+ spin_unlock_irqrestore(&phba->hbalock, iflags);
+ hrq->RQ_rcv_buf++;
+ fc_hdr = (struct fc_frame_header *)dma_buf->hbuf.virt;
+
+ /* Just some basic sanity checks on FCP Command frame */
+ fctl = (fc_hdr->fh_f_ctl[0] << 16 |
+ fc_hdr->fh_f_ctl[1] << 8 |
+ fc_hdr->fh_f_ctl[2]);
+ if (((fctl &
+ (FC_FC_FIRST_SEQ | FC_FC_END_SEQ | FC_FC_SEQ_INIT)) !=
+ (FC_FC_FIRST_SEQ | FC_FC_END_SEQ | FC_FC_SEQ_INIT)) ||
+ (fc_hdr->fh_seq_cnt != 0)) /* 0 byte swapped is still 0 */
+ goto drop;
+
+ if (fc_hdr->fh_type == FC_TYPE_FCP) {
+ dma_buf->bytes_recv = bf_get(lpfc_rcqe_length, rcqe);
+ lpfc_nvmet_unsol_fcp_event(
+ phba, phba->sli4_hba.els_wq->pring, dma_buf,
+ cq->assoc_qp->isr_timestamp);
+ return false;
+ }
+drop:
+ lpfc_in_buf_free(phba, &dma_buf->dbuf);
+ break;
+ case FC_STATUS_INSUFF_BUF_NEED_BUF:
+ case FC_STATUS_INSUFF_BUF_FRM_DISC:
+ hrq->RQ_no_posted_buf++;
+ /* Post more buffers if possible */
+ spin_lock_irqsave(&phba->hbalock, iflags);
+ phba->hba_flag |= HBA_POST_RECEIVE_BUFFER;
+ spin_unlock_irqrestore(&phba->hbalock, iflags);
+ workposted = true;
+ break;
+ }
+out:
+ return workposted;
+}
+
+/**
+ * lpfc_sli4_fp_handle_cqe - Process fast-path work queue completion entry
+ * @cq: Pointer to the completion queue.
+ * @eqe: Pointer to fast-path completion queue entry.
+ *
+ * This routine process a fast-path work queue completion entry from fast-path
* event queue for FCP command response completion.
**/
static int
-lpfc_sli4_fp_handle_wcqe(struct lpfc_hba *phba, struct lpfc_queue *cq,
+lpfc_sli4_fp_handle_cqe(struct lpfc_hba *phba, struct lpfc_queue *cq,
struct lpfc_cqe *cqe)
{
struct lpfc_wcqe_release wcqe;
/* Check and process for different type of WCQE and dispatch */
switch (bf_get(lpfc_wcqe_c_code, &wcqe)) {
case CQE_CODE_COMPL_WQE:
+ case CQE_CODE_NVME_ERSP:
cq->CQ_wq++;
/* Process the WQ complete event */
phba->last_completion_time = jiffies;
- lpfc_sli4_fp_handle_fcp_wcqe(phba, cq,
+ if ((cq->subtype == LPFC_FCP) || (cq->subtype == LPFC_NVME))
+ lpfc_sli4_fp_handle_fcp_wcqe(phba, cq,
+ (struct lpfc_wcqe_complete *)&wcqe);
+ if (cq->subtype == LPFC_NVME_LS)
+ lpfc_sli4_fp_handle_fcp_wcqe(phba, cq,
(struct lpfc_wcqe_complete *)&wcqe);
break;
case CQE_CODE_RELEASE_WQE:
workposted = lpfc_sli4_sp_handle_abort_xri_wcqe(phba, cq,
(struct sli4_wcqe_xri_aborted *)&wcqe);
break;
+ case CQE_CODE_RECEIVE_V1:
+ case CQE_CODE_RECEIVE:
+ phba->last_completion_time = jiffies;
+ if (cq->subtype == LPFC_NVMET) {
+ workposted = lpfc_sli4_nvmet_handle_rcqe(
+ phba, cq, (struct lpfc_rcqe *)&wcqe);
+ }
+ break;
default:
lpfc_printf_log(phba, KERN_ERR, LOG_SLI,
- "0144 Not a valid WCQE code: x%x\n",
+ "0144 Not a valid CQE code: x%x\n",
bf_get(lpfc_wcqe_c_code, &wcqe));
break;
}
lpfc_sli4_hba_handle_eqe(struct lpfc_hba *phba, struct lpfc_eqe *eqe,
uint32_t qidx)
{
- struct lpfc_queue *cq;
+ struct lpfc_queue *cq = NULL;
struct lpfc_cqe *cqe;
bool workposted = false;
- uint16_t cqid;
+ uint16_t cqid, id;
int ecount = 0;
if (unlikely(bf_get_le32(lpfc_eqe_major_code, eqe) != 0)) {
/* Get the reference to the corresponding CQ */
cqid = bf_get_le32(lpfc_eqe_resource_id, eqe);
- /* Check if this is a Slow path event */
- if (unlikely(cqid != phba->sli4_hba.fcp_cq_map[qidx])) {
- lpfc_sli4_sp_handle_eqe(phba, eqe,
- phba->sli4_hba.hba_eq[qidx]);
- return;
+ if (phba->cfg_nvmet_mrq && phba->sli4_hba.nvmet_cqset) {
+ id = phba->sli4_hba.nvmet_cqset[0]->queue_id;
+ if ((cqid >= id) && (cqid < (id + phba->cfg_nvmet_mrq))) {
+ /* Process NVMET unsol rcv */
+ cq = phba->sli4_hba.nvmet_cqset[cqid - id];
+ goto process_cq;
+ }
}
- if (unlikely(!phba->sli4_hba.fcp_cq)) {
- lpfc_printf_log(phba, KERN_WARNING, LOG_SLI,
- "3146 Fast-path completion queues "
- "does not exist\n");
- return;
+ if (phba->sli4_hba.nvme_cq_map &&
+ (cqid == phba->sli4_hba.nvme_cq_map[qidx])) {
+ /* Process NVME / NVMET command completion */
+ cq = phba->sli4_hba.nvme_cq[qidx];
+ goto process_cq;
}
- cq = phba->sli4_hba.fcp_cq[qidx];
- if (unlikely(!cq)) {
- if (phba->sli.sli_flag & LPFC_SLI_ACTIVE)
- lpfc_printf_log(phba, KERN_ERR, LOG_SLI,
- "0367 Fast-path completion queue "
- "(%d) does not exist\n", qidx);
+
+ if (phba->sli4_hba.fcp_cq_map &&
+ (cqid == phba->sli4_hba.fcp_cq_map[qidx])) {
+ /* Process FCP command completion */
+ cq = phba->sli4_hba.fcp_cq[qidx];
+ goto process_cq;
+ }
+
+ if (phba->sli4_hba.nvmels_cq &&
+ (cqid == phba->sli4_hba.nvmels_cq->queue_id)) {
+ /* Process NVME unsol rcv */
+ cq = phba->sli4_hba.nvmels_cq;
+ }
+
+ /* Otherwise this is a Slow path event */
+ if (cq == NULL) {
+ lpfc_sli4_sp_handle_eqe(phba, eqe, phba->sli4_hba.hba_eq[qidx]);
return;
}
+process_cq:
if (unlikely(cqid != cq->queue_id)) {
lpfc_printf_log(phba, KERN_ERR, LOG_SLI,
"0368 Miss-matched fast-path completion "
return;
}
+ /* Save EQ associated with this CQ */
+ cq->assoc_qp = phba->sli4_hba.hba_eq[qidx];
+
/* Process all the entries to the CQ */
while ((cqe = lpfc_sli4_cq_get(cq))) {
- workposted |= lpfc_sli4_fp_handle_wcqe(phba, cq, cqe);
+ workposted |= lpfc_sli4_fp_handle_cqe(phba, cq, cqe);
if (!(++ecount % cq->entry_repost))
lpfc_sli4_cq_release(cq, LPFC_QUEUE_NOARM);
}
/* Process all the entries to the OAS CQ */
while ((cqe = lpfc_sli4_cq_get(cq))) {
- workposted |= lpfc_sli4_fp_handle_wcqe(phba, cq, cqe);
+ workposted |= lpfc_sli4_fp_handle_cqe(phba, cq, cqe);
if (!(++ecount % cq->entry_repost))
lpfc_sli4_cq_release(cq, LPFC_QUEUE_NOARM);
}
lpfc_sli4_fof_intr_handler(int irq, void *dev_id)
{
struct lpfc_hba *phba;
- struct lpfc_fcp_eq_hdl *fcp_eq_hdl;
+ struct lpfc_hba_eq_hdl *hba_eq_hdl;
struct lpfc_queue *eq;
struct lpfc_eqe *eqe;
unsigned long iflag;
int ecount = 0;
/* Get the driver's phba structure from the dev_id */
- fcp_eq_hdl = (struct lpfc_fcp_eq_hdl *)dev_id;
- phba = fcp_eq_hdl->phba;
+ hba_eq_hdl = (struct lpfc_hba_eq_hdl *)dev_id;
+ phba = hba_eq_hdl->phba;
if (unlikely(!phba))
return IRQ_NONE;
lpfc_sli4_hba_intr_handler(int irq, void *dev_id)
{
struct lpfc_hba *phba;
- struct lpfc_fcp_eq_hdl *fcp_eq_hdl;
+ struct lpfc_hba_eq_hdl *hba_eq_hdl;
struct lpfc_queue *fpeq;
struct lpfc_eqe *eqe;
unsigned long iflag;
int ecount = 0;
- int fcp_eqidx;
+ int hba_eqidx;
/* Get the driver's phba structure from the dev_id */
- fcp_eq_hdl = (struct lpfc_fcp_eq_hdl *)dev_id;
- phba = fcp_eq_hdl->phba;
- fcp_eqidx = fcp_eq_hdl->idx;
+ hba_eq_hdl = (struct lpfc_hba_eq_hdl *)dev_id;
+ phba = hba_eq_hdl->phba;
+ hba_eqidx = hba_eq_hdl->idx;
if (unlikely(!phba))
return IRQ_NONE;
return IRQ_NONE;
/* Get to the EQ struct associated with this vector */
- fpeq = phba->sli4_hba.hba_eq[fcp_eqidx];
+ fpeq = phba->sli4_hba.hba_eq[hba_eqidx];
if (unlikely(!fpeq))
return IRQ_NONE;
+#ifdef CONFIG_SCSI_LPFC_DEBUG_FS
+ if (phba->ktime_on)
+ fpeq->isr_timestamp = ktime_get_ns();
+#endif
+
if (lpfc_fcp_look_ahead) {
- if (atomic_dec_and_test(&fcp_eq_hdl->fcp_eq_in_use))
+ if (atomic_dec_and_test(&hba_eq_hdl->hba_eq_in_use))
lpfc_sli4_eq_clr_intr(fpeq);
else {
- atomic_inc(&fcp_eq_hdl->fcp_eq_in_use);
+ atomic_inc(&hba_eq_hdl->hba_eq_in_use);
return IRQ_NONE;
}
}
lpfc_sli4_eq_flush(phba, fpeq);
spin_unlock_irqrestore(&phba->hbalock, iflag);
if (lpfc_fcp_look_ahead)
- atomic_inc(&fcp_eq_hdl->fcp_eq_in_use);
+ atomic_inc(&hba_eq_hdl->hba_eq_in_use);
return IRQ_NONE;
}
if (eqe == NULL)
break;
- lpfc_sli4_hba_handle_eqe(phba, eqe, fcp_eqidx);
+ lpfc_sli4_hba_handle_eqe(phba, eqe, hba_eqidx);
if (!(++ecount % fpeq->entry_repost))
lpfc_sli4_eq_release(fpeq, LPFC_QUEUE_NOARM);
fpeq->EQ_processed++;
fpeq->EQ_no_entry++;
if (lpfc_fcp_look_ahead) {
- atomic_inc(&fcp_eq_hdl->fcp_eq_in_use);
+ atomic_inc(&hba_eq_hdl->hba_eq_in_use);
return IRQ_NONE;
}
}
if (lpfc_fcp_look_ahead)
- atomic_inc(&fcp_eq_hdl->fcp_eq_in_use);
+ atomic_inc(&hba_eq_hdl->hba_eq_in_use);
+
return IRQ_HANDLED;
} /* lpfc_sli4_fp_intr_handler */
struct lpfc_hba *phba;
irqreturn_t hba_irq_rc;
bool hba_handled = false;
- int fcp_eqidx;
+ int qidx;
/* Get the driver's phba structure from the dev_id */
phba = (struct lpfc_hba *)dev_id;
/*
* Invoke fast-path host attention interrupt handling as appropriate.
*/
- for (fcp_eqidx = 0; fcp_eqidx < phba->cfg_fcp_io_channel; fcp_eqidx++) {
+ for (qidx = 0; qidx < phba->io_channel_irqs; qidx++) {
hba_irq_rc = lpfc_sli4_hba_intr_handler(irq,
- &phba->sli4_hba.fcp_eq_hdl[fcp_eqidx]);
+ &phba->sli4_hba.hba_eq_hdl[qidx]);
if (hba_irq_rc == IRQ_HANDLED)
hba_handled |= true;
}
if (phba->cfg_fof) {
hba_irq_rc = lpfc_sli4_fof_intr_handler(irq,
- &phba->sli4_hba.fcp_eq_hdl[0]);
+ &phba->sli4_hba.hba_eq_hdl[qidx]);
if (hba_irq_rc == IRQ_HANDLED)
hba_handled |= true;
}
dmabuf->virt, dmabuf->phys);
kfree(dmabuf);
}
+ if (queue->rqbp) {
+ lpfc_free_rq_buffer(queue->phba, queue);
+ kfree(queue->rqbp);
+ }
+ kfree(queue->pring);
kfree(queue);
return;
}
return NULL;
queue->page_count = (ALIGN(entry_size * entry_count,
hw_page_size))/hw_page_size;
+
+ /* If needed, Adjust page count to match the max the adapter supports */
+ if (queue->page_count > phba->sli4_hba.pc_sli4_params.wqpcnt)
+ queue->page_count = phba->sli4_hba.pc_sli4_params.wqpcnt;
+
INIT_LIST_HEAD(&queue->list);
+ INIT_LIST_HEAD(&queue->wq_list);
INIT_LIST_HEAD(&queue->page_list);
INIT_LIST_HEAD(&queue->child_list);
for (x = 0, total_qe_count = 0; x < queue->page_count; x++) {
}
/**
- * lpfc_modify_fcp_eq_delay - Modify Delay Multiplier on FCP EQs
+ * lpfc_modify_hba_eq_delay - Modify Delay Multiplier on FCP EQs
* @phba: HBA structure that indicates port to create a queue on.
* @startq: The starting FCP EQ to modify
*
* fails this function will return -ENXIO.
**/
int
-lpfc_modify_fcp_eq_delay(struct lpfc_hba *phba, uint32_t startq)
+lpfc_modify_hba_eq_delay(struct lpfc_hba *phba, uint32_t startq)
{
struct lpfc_mbx_modify_eq_delay *eq_delay;
LPFC_MBOXQ_t *mbox;
int cnt, rc, length, status = 0;
uint32_t shdr_status, shdr_add_status;
uint32_t result;
- int fcp_eqidx;
+ int qidx;
union lpfc_sli4_cfg_shdr *shdr;
uint16_t dmult;
- if (startq >= phba->cfg_fcp_io_channel)
+ if (startq >= phba->io_channel_irqs)
return 0;
mbox = mempool_alloc(phba->mbox_mem_pool, GFP_KERNEL);
eq_delay = &mbox->u.mqe.un.eq_delay;
/* Calculate delay multiper from maximum interrupt per second */
- result = phba->cfg_fcp_imax / phba->cfg_fcp_io_channel;
- if (result > LPFC_DMULT_CONST)
+ result = phba->cfg_fcp_imax / phba->io_channel_irqs;
+ if (result > LPFC_DMULT_CONST || result == 0)
dmult = 0;
else
dmult = LPFC_DMULT_CONST/result - 1;
cnt = 0;
- for (fcp_eqidx = startq; fcp_eqidx < phba->cfg_fcp_io_channel;
- fcp_eqidx++) {
- eq = phba->sli4_hba.hba_eq[fcp_eqidx];
+ for (qidx = startq; qidx < phba->io_channel_irqs; qidx++) {
+ eq = phba->sli4_hba.hba_eq[qidx];
if (!eq)
continue;
eq_delay->u.request.eq[cnt].eq_id = eq->queue_id;
switch (cq->entry_count) {
default:
lpfc_printf_log(phba, KERN_ERR, LOG_SLI,
- "0361 Unsupported CQ count. (%d)\n",
- cq->entry_count);
+ "0361 Unsupported CQ count: "
+ "entry cnt %d sz %d pg cnt %d repost %d\n",
+ cq->entry_count, cq->entry_size,
+ cq->page_count, cq->entry_repost);
if (cq->entry_count < 256) {
status = -EINVAL;
goto out;
return status;
}
+/**
+ * lpfc_cq_create_set - Create a set of Completion Queues on the HBA for MRQ
+ * @phba: HBA structure that indicates port to create a queue on.
+ * @cqp: The queue structure array to use to create the completion queues.
+ * @eqp: The event queue array to bind these completion queues to.
+ *
+ * This function creates a set of completion queue, s to support MRQ
+ * as detailed in @cqp, on a port,
+ * described by @phba by sending a CREATE_CQ_SET mailbox command to the HBA.
+ *
+ * The @phba struct is used to send mailbox command to HBA. The @cq struct
+ * is used to get the entry count and entry size that are necessary to
+ * determine the number of pages to allocate and use for this queue. The @eq
+ * is used to indicate which event queue to bind this completion queue to. This
+ * function will send the CREATE_CQ_SET mailbox command to the HBA to setup the
+ * completion queue. This function is asynchronous and will wait for the mailbox
+ * command to finish before continuing.
+ *
+ * On success this function will return a zero. If unable to allocate enough
+ * memory this function will return -ENOMEM. If the queue create mailbox command
+ * fails this function will return -ENXIO.
+ **/
+int
+lpfc_cq_create_set(struct lpfc_hba *phba, struct lpfc_queue **cqp,
+ struct lpfc_queue **eqp, uint32_t type, uint32_t subtype)
+{
+ struct lpfc_queue *cq;
+ struct lpfc_queue *eq;
+ struct lpfc_mbx_cq_create_set *cq_set;
+ struct lpfc_dmabuf *dmabuf;
+ LPFC_MBOXQ_t *mbox;
+ int rc, length, alloclen, status = 0;
+ int cnt, idx, numcq, page_idx = 0;
+ uint32_t shdr_status, shdr_add_status;
+ union lpfc_sli4_cfg_shdr *shdr;
+ uint32_t hw_page_size = phba->sli4_hba.pc_sli4_params.if_page_sz;
+
+ /* sanity check on queue memory */
+ numcq = phba->cfg_nvmet_mrq;
+ if (!cqp || !eqp || !numcq)
+ return -ENODEV;
+ if (!phba->sli4_hba.pc_sli4_params.supported)
+ hw_page_size = SLI4_PAGE_SIZE;
+
+ mbox = mempool_alloc(phba->mbox_mem_pool, GFP_KERNEL);
+ if (!mbox)
+ return -ENOMEM;
+
+ length = sizeof(struct lpfc_mbx_cq_create_set);
+ length += ((numcq * cqp[0]->page_count) *
+ sizeof(struct dma_address));
+ alloclen = lpfc_sli4_config(phba, mbox, LPFC_MBOX_SUBSYSTEM_FCOE,
+ LPFC_MBOX_OPCODE_FCOE_CQ_CREATE_SET, length,
+ LPFC_SLI4_MBX_NEMBED);
+ if (alloclen < length) {
+ lpfc_printf_log(phba, KERN_ERR, LOG_SLI,
+ "3098 Allocated DMA memory size (%d) is "
+ "less than the requested DMA memory size "
+ "(%d)\n", alloclen, length);
+ status = -ENOMEM;
+ goto out;
+ }
+ cq_set = mbox->sge_array->addr[0];
+ shdr = (union lpfc_sli4_cfg_shdr *)&cq_set->cfg_shdr;
+ bf_set(lpfc_mbox_hdr_version, &shdr->request, 0);
+
+ for (idx = 0; idx < numcq; idx++) {
+ cq = cqp[idx];
+ eq = eqp[idx];
+ if (!cq || !eq) {
+ status = -ENOMEM;
+ goto out;
+ }
+
+ switch (idx) {
+ case 0:
+ bf_set(lpfc_mbx_cq_create_set_page_size,
+ &cq_set->u.request,
+ (hw_page_size / SLI4_PAGE_SIZE));
+ bf_set(lpfc_mbx_cq_create_set_num_pages,
+ &cq_set->u.request, cq->page_count);
+ bf_set(lpfc_mbx_cq_create_set_evt,
+ &cq_set->u.request, 1);
+ bf_set(lpfc_mbx_cq_create_set_valid,
+ &cq_set->u.request, 1);
+ bf_set(lpfc_mbx_cq_create_set_cqe_size,
+ &cq_set->u.request, 0);
+ bf_set(lpfc_mbx_cq_create_set_num_cq,
+ &cq_set->u.request, numcq);
+ switch (cq->entry_count) {
+ default:
+ lpfc_printf_log(phba, KERN_ERR, LOG_SLI,
+ "3118 Bad CQ count. (%d)\n",
+ cq->entry_count);
+ if (cq->entry_count < 256) {
+ status = -EINVAL;
+ goto out;
+ }
+ /* otherwise default to smallest (drop thru) */
+ case 256:
+ bf_set(lpfc_mbx_cq_create_set_cqe_cnt,
+ &cq_set->u.request, LPFC_CQ_CNT_256);
+ break;
+ case 512:
+ bf_set(lpfc_mbx_cq_create_set_cqe_cnt,
+ &cq_set->u.request, LPFC_CQ_CNT_512);
+ break;
+ case 1024:
+ bf_set(lpfc_mbx_cq_create_set_cqe_cnt,
+ &cq_set->u.request, LPFC_CQ_CNT_1024);
+ break;
+ }
+ bf_set(lpfc_mbx_cq_create_set_eq_id0,
+ &cq_set->u.request, eq->queue_id);
+ break;
+ case 1:
+ bf_set(lpfc_mbx_cq_create_set_eq_id1,
+ &cq_set->u.request, eq->queue_id);
+ break;
+ case 2:
+ bf_set(lpfc_mbx_cq_create_set_eq_id2,
+ &cq_set->u.request, eq->queue_id);
+ break;
+ case 3:
+ bf_set(lpfc_mbx_cq_create_set_eq_id3,
+ &cq_set->u.request, eq->queue_id);
+ break;
+ case 4:
+ bf_set(lpfc_mbx_cq_create_set_eq_id4,
+ &cq_set->u.request, eq->queue_id);
+ break;
+ case 5:
+ bf_set(lpfc_mbx_cq_create_set_eq_id5,
+ &cq_set->u.request, eq->queue_id);
+ break;
+ case 6:
+ bf_set(lpfc_mbx_cq_create_set_eq_id6,
+ &cq_set->u.request, eq->queue_id);
+ break;
+ case 7:
+ bf_set(lpfc_mbx_cq_create_set_eq_id7,
+ &cq_set->u.request, eq->queue_id);
+ break;
+ case 8:
+ bf_set(lpfc_mbx_cq_create_set_eq_id8,
+ &cq_set->u.request, eq->queue_id);
+ break;
+ case 9:
+ bf_set(lpfc_mbx_cq_create_set_eq_id9,
+ &cq_set->u.request, eq->queue_id);
+ break;
+ case 10:
+ bf_set(lpfc_mbx_cq_create_set_eq_id10,
+ &cq_set->u.request, eq->queue_id);
+ break;
+ case 11:
+ bf_set(lpfc_mbx_cq_create_set_eq_id11,
+ &cq_set->u.request, eq->queue_id);
+ break;
+ case 12:
+ bf_set(lpfc_mbx_cq_create_set_eq_id12,
+ &cq_set->u.request, eq->queue_id);
+ break;
+ case 13:
+ bf_set(lpfc_mbx_cq_create_set_eq_id13,
+ &cq_set->u.request, eq->queue_id);
+ break;
+ case 14:
+ bf_set(lpfc_mbx_cq_create_set_eq_id14,
+ &cq_set->u.request, eq->queue_id);
+ break;
+ case 15:
+ bf_set(lpfc_mbx_cq_create_set_eq_id15,
+ &cq_set->u.request, eq->queue_id);
+ break;
+ }
+
+ /* link the cq onto the parent eq child list */
+ list_add_tail(&cq->list, &eq->child_list);
+ /* Set up completion queue's type and subtype */
+ cq->type = type;
+ cq->subtype = subtype;
+ cq->assoc_qid = eq->queue_id;
+ cq->host_index = 0;
+ cq->hba_index = 0;
+
+ rc = 0;
+ list_for_each_entry(dmabuf, &cq->page_list, list) {
+ memset(dmabuf->virt, 0, hw_page_size);
+ cnt = page_idx + dmabuf->buffer_tag;
+ cq_set->u.request.page[cnt].addr_lo =
+ putPaddrLow(dmabuf->phys);
+ cq_set->u.request.page[cnt].addr_hi =
+ putPaddrHigh(dmabuf->phys);
+ rc++;
+ }
+ page_idx += rc;
+ }
+
+ rc = lpfc_sli_issue_mbox(phba, mbox, MBX_POLL);
+
+ /* The IOCTL status is embedded in the mailbox subheader. */
+ shdr_status = bf_get(lpfc_mbox_hdr_status, &shdr->response);
+ shdr_add_status = bf_get(lpfc_mbox_hdr_add_status, &shdr->response);
+ if (shdr_status || shdr_add_status || rc) {
+ lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+ "3119 CQ_CREATE_SET mailbox failed with "
+ "status x%x add_status x%x, mbx status x%x\n",
+ shdr_status, shdr_add_status, rc);
+ status = -ENXIO;
+ goto out;
+ }
+ rc = bf_get(lpfc_mbx_cq_create_set_base_id, &cq_set->u.response);
+ if (rc == 0xFFFF) {
+ status = -ENXIO;
+ goto out;
+ }
+
+ for (idx = 0; idx < numcq; idx++) {
+ cq = cqp[idx];
+ cq->queue_id = rc + idx;
+ }
+
+out:
+ lpfc_sli4_mbox_cmd_free(phba, mbox);
+ return status;
+}
+
/**
* lpfc_mq_create_fb_init - Send MCC_CREATE without async events registration
* @phba: HBA structure that indicates port to create a queue on.
LPFC_WQ_WQE_SIZE_128);
bf_set(lpfc_mbx_wq_create_page_size,
&wq_create->u.request_1,
- (PAGE_SIZE/SLI4_PAGE_SIZE));
+ LPFC_WQ_PAGE_SIZE_4096);
page = wq_create->u.request_1.page;
break;
}
LPFC_WQ_WQE_SIZE_128);
break;
}
- bf_set(lpfc_mbx_wq_create_page_size, &wq_create->u.request_1,
- (PAGE_SIZE/SLI4_PAGE_SIZE));
+ bf_set(lpfc_mbx_wq_create_page_size,
+ &wq_create->u.request_1,
+ LPFC_WQ_PAGE_SIZE_4096);
page = wq_create->u.request_1.page;
break;
default:
wq->db_format = LPFC_DB_LIST_FORMAT;
wq->db_regaddr = phba->sli4_hba.WQDBregaddr;
}
+ wq->pring = kzalloc(sizeof(struct lpfc_sli_ring), GFP_KERNEL);
+ if (wq->pring == NULL) {
+ status = -ENOMEM;
+ goto out;
+ }
wq->type = LPFC_WQ;
wq->assoc_qid = cq->queue_id;
wq->subtype = subtype;
LPFC_RQE_SIZE_8);
bf_set(lpfc_rq_context_page_size,
&rq_create->u.request.context,
- (PAGE_SIZE/SLI4_PAGE_SIZE));
+ LPFC_RQ_PAGE_SIZE_4096);
} else {
switch (hrq->entry_count) {
default:
return status;
}
+/**
+ * lpfc_mrq_create - Create MRQ Receive Queues on the HBA
+ * @phba: HBA structure that indicates port to create a queue on.
+ * @hrqp: The queue structure array to use to create the header receive queues.
+ * @drqp: The queue structure array to use to create the data receive queues.
+ * @cqp: The completion queue array to bind these receive queues to.
+ *
+ * This function creates a receive buffer queue pair , as detailed in @hrq and
+ * @drq, on a port, described by @phba by sending a RQ_CREATE mailbox command
+ * to the HBA.
+ *
+ * The @phba struct is used to send mailbox command to HBA. The @drq and @hrq
+ * struct is used to get the entry count that is necessary to determine the
+ * number of pages to use for this queue. The @cq is used to indicate which
+ * completion queue to bind received buffers that are posted to these queues to.
+ * This function will send the RQ_CREATE mailbox command to the HBA to setup the
+ * receive queue pair. This function is asynchronous and will wait for the
+ * mailbox command to finish before continuing.
+ *
+ * On success this function will return a zero. If unable to allocate enough
+ * memory this function will return -ENOMEM. If the queue create mailbox command
+ * fails this function will return -ENXIO.
+ **/
+int
+lpfc_mrq_create(struct lpfc_hba *phba, struct lpfc_queue **hrqp,
+ struct lpfc_queue **drqp, struct lpfc_queue **cqp,
+ uint32_t subtype)
+{
+ struct lpfc_queue *hrq, *drq, *cq;
+ struct lpfc_mbx_rq_create_v2 *rq_create;
+ struct lpfc_dmabuf *dmabuf;
+ LPFC_MBOXQ_t *mbox;
+ int rc, length, alloclen, status = 0;
+ int cnt, idx, numrq, page_idx = 0;
+ uint32_t shdr_status, shdr_add_status;
+ union lpfc_sli4_cfg_shdr *shdr;
+ uint32_t hw_page_size = phba->sli4_hba.pc_sli4_params.if_page_sz;
+
+ numrq = phba->cfg_nvmet_mrq;
+ /* sanity check on array memory */
+ if (!hrqp || !drqp || !cqp || !numrq)
+ return -ENODEV;
+ if (!phba->sli4_hba.pc_sli4_params.supported)
+ hw_page_size = SLI4_PAGE_SIZE;
+
+ mbox = mempool_alloc(phba->mbox_mem_pool, GFP_KERNEL);
+ if (!mbox)
+ return -ENOMEM;
+
+ length = sizeof(struct lpfc_mbx_rq_create_v2);
+ length += ((2 * numrq * hrqp[0]->page_count) *
+ sizeof(struct dma_address));
+
+ alloclen = lpfc_sli4_config(phba, mbox, LPFC_MBOX_SUBSYSTEM_FCOE,
+ LPFC_MBOX_OPCODE_FCOE_RQ_CREATE, length,
+ LPFC_SLI4_MBX_NEMBED);
+ if (alloclen < length) {
+ lpfc_printf_log(phba, KERN_ERR, LOG_SLI,
+ "3099 Allocated DMA memory size (%d) is "
+ "less than the requested DMA memory size "
+ "(%d)\n", alloclen, length);
+ status = -ENOMEM;
+ goto out;
+ }
+
+
+
+ rq_create = mbox->sge_array->addr[0];
+ shdr = (union lpfc_sli4_cfg_shdr *)&rq_create->cfg_shdr;
+
+ bf_set(lpfc_mbox_hdr_version, &shdr->request, LPFC_Q_CREATE_VERSION_2);
+ cnt = 0;
+
+ for (idx = 0; idx < numrq; idx++) {
+ hrq = hrqp[idx];
+ drq = drqp[idx];
+ cq = cqp[idx];
+
+ if (hrq->entry_count != drq->entry_count) {
+ status = -EINVAL;
+ goto out;
+ }
+
+ /* sanity check on queue memory */
+ if (!hrq || !drq || !cq) {
+ status = -ENODEV;
+ goto out;
+ }
+
+ if (idx == 0) {
+ bf_set(lpfc_mbx_rq_create_num_pages,
+ &rq_create->u.request,
+ hrq->page_count);
+ bf_set(lpfc_mbx_rq_create_rq_cnt,
+ &rq_create->u.request, (numrq * 2));
+ bf_set(lpfc_mbx_rq_create_dnb, &rq_create->u.request,
+ 1);
+ bf_set(lpfc_rq_context_base_cq,
+ &rq_create->u.request.context,
+ cq->queue_id);
+ bf_set(lpfc_rq_context_data_size,
+ &rq_create->u.request.context,
+ LPFC_DATA_BUF_SIZE);
+ bf_set(lpfc_rq_context_hdr_size,
+ &rq_create->u.request.context,
+ LPFC_HDR_BUF_SIZE);
+ bf_set(lpfc_rq_context_rqe_count_1,
+ &rq_create->u.request.context,
+ hrq->entry_count);
+ bf_set(lpfc_rq_context_rqe_size,
+ &rq_create->u.request.context,
+ LPFC_RQE_SIZE_8);
+ bf_set(lpfc_rq_context_page_size,
+ &rq_create->u.request.context,
+ (PAGE_SIZE/SLI4_PAGE_SIZE));
+ }
+ rc = 0;
+ list_for_each_entry(dmabuf, &hrq->page_list, list) {
+ memset(dmabuf->virt, 0, hw_page_size);
+ cnt = page_idx + dmabuf->buffer_tag;
+ rq_create->u.request.page[cnt].addr_lo =
+ putPaddrLow(dmabuf->phys);
+ rq_create->u.request.page[cnt].addr_hi =
+ putPaddrHigh(dmabuf->phys);
+ rc++;
+ }
+ page_idx += rc;
+
+ rc = 0;
+ list_for_each_entry(dmabuf, &drq->page_list, list) {
+ memset(dmabuf->virt, 0, hw_page_size);
+ cnt = page_idx + dmabuf->buffer_tag;
+ rq_create->u.request.page[cnt].addr_lo =
+ putPaddrLow(dmabuf->phys);
+ rq_create->u.request.page[cnt].addr_hi =
+ putPaddrHigh(dmabuf->phys);
+ rc++;
+ }
+ page_idx += rc;
+
+ hrq->db_format = LPFC_DB_RING_FORMAT;
+ hrq->db_regaddr = phba->sli4_hba.RQDBregaddr;
+ hrq->type = LPFC_HRQ;
+ hrq->assoc_qid = cq->queue_id;
+ hrq->subtype = subtype;
+ hrq->host_index = 0;
+ hrq->hba_index = 0;
+
+ drq->db_format = LPFC_DB_RING_FORMAT;
+ drq->db_regaddr = phba->sli4_hba.RQDBregaddr;
+ drq->type = LPFC_DRQ;
+ drq->assoc_qid = cq->queue_id;
+ drq->subtype = subtype;
+ drq->host_index = 0;
+ drq->hba_index = 0;
+
+ list_add_tail(&hrq->list, &cq->child_list);
+ list_add_tail(&drq->list, &cq->child_list);
+ }
+
+ rc = lpfc_sli_issue_mbox(phba, mbox, MBX_POLL);
+ /* The IOCTL status is embedded in the mailbox subheader. */
+ shdr_status = bf_get(lpfc_mbox_hdr_status, &shdr->response);
+ shdr_add_status = bf_get(lpfc_mbox_hdr_add_status, &shdr->response);
+ if (shdr_status || shdr_add_status || rc) {
+ lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+ "3120 RQ_CREATE mailbox failed with "
+ "status x%x add_status x%x, mbx status x%x\n",
+ shdr_status, shdr_add_status, rc);
+ status = -ENXIO;
+ goto out;
+ }
+ rc = bf_get(lpfc_mbx_rq_create_q_id, &rq_create->u.response);
+ if (rc == 0xFFFF) {
+ status = -ENXIO;
+ goto out;
+ }
+
+ /* Initialize all RQs with associated queue id */
+ for (idx = 0; idx < numrq; idx++) {
+ hrq = hrqp[idx];
+ hrq->queue_id = rc + (2 * idx);
+ drq = drqp[idx];
+ drq->queue_id = rc + (2 * idx) + 1;
+ }
+
+out:
+ lpfc_sli4_mbox_cmd_free(phba, mbox);
+ return status;
+}
+
/**
* lpfc_eq_destroy - Destroy an event Queue on the HBA
* @eq: The queue structure associated with the queue to destroy.
}
/**
- * lpfc_sli4_post_els_sgl_list - post a block of ELS sgls to the port.
+ * lpfc_sli4_post_sgl_list - post a block of ELS sgls to the port.
* @phba: pointer to lpfc hba data structure.
* @post_sgl_list: pointer to els sgl entry list.
* @count: number of els sgl entries on the list.
* stopped.
**/
static int
-lpfc_sli4_post_els_sgl_list(struct lpfc_hba *phba,
+lpfc_sli4_post_sgl_list(struct lpfc_hba *phba,
struct list_head *post_sgl_list,
int post_cnt)
{
uint32_t shdr_status, shdr_add_status;
union lpfc_sli4_cfg_shdr *shdr;
- reqlen = phba->sli4_hba.els_xri_cnt * sizeof(struct sgl_page_pairs) +
+ reqlen = post_cnt * sizeof(struct sgl_page_pairs) +
sizeof(union lpfc_sli4_cfg_shdr) + sizeof(uint32_t);
if (reqlen > SLI4_PAGE_SIZE) {
- lpfc_printf_log(phba, KERN_WARNING, LOG_INIT,
+ lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
"2559 Block sgl registration required DMA "
"size (%d) great than a page\n", reqlen);
return -ENOMEM;
}
+
mbox = mempool_alloc(phba->mbox_mem_pool, GFP_KERNEL);
if (!mbox)
return -ENOMEM;
/* Complete initialization and perform endian conversion. */
bf_set(lpfc_post_sgl_pages_xri, sgl, xritag_start);
- bf_set(lpfc_post_sgl_pages_xricnt, sgl, phba->sli4_hba.els_xri_cnt);
+ bf_set(lpfc_post_sgl_pages_xricnt, sgl, post_cnt);
sgl->word0 = cpu_to_le32(sgl->word0);
+
if (!phba->sli4_hba.intr_enable)
rc = lpfc_sli_issue_mbox(phba, mbox, MBX_POLL);
else {
return rc;
}
+static char *lpfc_rctl_names[] = FC_RCTL_NAMES_INIT;
+static char *lpfc_type_names[] = FC_TYPE_NAMES_INIT;
+
/**
* lpfc_fc_frame_check - Check that this frame is a valid frame to handle
* @phba: pointer to lpfc_hba struct that the frame was received on
lpfc_fc_frame_check(struct lpfc_hba *phba, struct fc_frame_header *fc_hdr)
{
/* make rctl_names static to save stack space */
- static char *rctl_names[] = FC_RCTL_NAMES_INIT;
- char *type_names[] = FC_TYPE_NAMES_INIT;
struct fc_vft_header *fc_vft_hdr;
uint32_t *header = (uint32_t *) fc_hdr;
case FC_TYPE_ELS:
case FC_TYPE_FCP:
case FC_TYPE_CT:
+ case FC_TYPE_NVME:
break;
case FC_TYPE_IP:
case FC_TYPE_ILS:
lpfc_printf_log(phba, KERN_INFO, LOG_ELS,
"2538 Received frame rctl:%s (x%x), type:%s (x%x), "
"frame Data:%08x %08x %08x %08x %08x %08x %08x\n",
- rctl_names[fc_hdr->fh_r_ctl], fc_hdr->fh_r_ctl,
- type_names[fc_hdr->fh_type], fc_hdr->fh_type,
+ lpfc_rctl_names[fc_hdr->fh_r_ctl], fc_hdr->fh_r_ctl,
+ lpfc_type_names[fc_hdr->fh_type], fc_hdr->fh_type,
be32_to_cpu(header[0]), be32_to_cpu(header[1]),
be32_to_cpu(header[2]), be32_to_cpu(header[3]),
be32_to_cpu(header[4]), be32_to_cpu(header[5]),
drop:
lpfc_printf_log(phba, KERN_WARNING, LOG_ELS,
"2539 Dropped frame rctl:%s type:%s\n",
- rctl_names[fc_hdr->fh_r_ctl],
- type_names[fc_hdr->fh_type]);
+ lpfc_rctl_names[fc_hdr->fh_r_ctl],
+ lpfc_type_names[fc_hdr->fh_type]);
return 1;
}
**/
static struct lpfc_vport *
lpfc_fc_frame_to_vport(struct lpfc_hba *phba, struct fc_frame_header *fc_hdr,
- uint16_t fcfi)
+ uint16_t fcfi, uint32_t did)
{
struct lpfc_vport **vports;
struct lpfc_vport *vport = NULL;
int i;
- uint32_t did = (fc_hdr->fh_d_id[0] << 16 |
- fc_hdr->fh_d_id[1] << 8 |
- fc_hdr->fh_d_id[2]);
if (did == Fabric_DID)
return phba->pport;
return phba->pport;
vports = lpfc_create_vport_work_array(phba);
- if (vports != NULL)
+ if (vports != NULL) {
for (i = 0; i <= phba->max_vpi && vports[i] != NULL; i++) {
if (phba->fcf.fcfi == fcfi &&
vports[i]->vfi == lpfc_fc_hdr_get_vfi(fc_hdr) &&
break;
}
}
+ }
lpfc_destroy_vport_work_array(phba, vports);
return vport;
}
* a BA_RJT.
*/
if ((fctl & FC_FC_EX_CTX) &&
- (lxri > lpfc_sli4_get_els_iocb_cnt(phba))) {
+ (lxri > lpfc_sli4_get_iocb_cnt(phba))) {
icmd->un.xseq64.w5.hcsw.Rctl = FC_RCTL_BA_RJT;
bf_set(lpfc_vndr_code, &icmd->un.bls_rsp, 0);
bf_set(lpfc_rsn_expln, &icmd->un.bls_rsp, FC_BA_RJT_INV_XID);
/* Initialize the first IOCB. */
first_iocbq->iocb.unsli3.rcvsli3.acc_len = 0;
first_iocbq->iocb.ulpStatus = IOSTAT_SUCCESS;
+ first_iocbq->vport = vport;
/* Check FC Header to see what TYPE of frame we are rcv'ing */
if (sli4_type_from_fc_hdr(fc_hdr) == FC_TYPE_ELS) {
return;
}
if (!lpfc_complete_unsol_iocb(phba,
- &phba->sli.ring[LPFC_ELS_RING],
+ phba->sli4_hba.els_wq->pring,
iocbq, fc_hdr->fh_r_ctl,
fc_hdr->fh_type))
lpfc_printf_log(phba, KERN_ERR, LOG_SLI,
* This function is called with no lock held. This function processes all
* the received buffers and gives it to upper layers when a received buffer
* indicates that it is the final frame in the sequence. The interrupt
- * service routine processes received buffers at interrupt contexts and adds
- * received dma buffers to the rb_pend_list queue and signals the worker thread.
+ * service routine processes received buffers at interrupt contexts.
* Worker thread calls lpfc_sli4_handle_received_buffer, which will call the
* appropriate receive function when the final frame in a sequence is received.
**/
/* Process each received buffer */
fc_hdr = (struct fc_frame_header *)dmabuf->hbuf.virt;
+
/* check to see if this a valid type of frame */
if (lpfc_fc_frame_check(phba, fc_hdr)) {
lpfc_in_buf_free(phba, &dmabuf->dbuf);
return;
}
+
if ((bf_get(lpfc_cqe_code,
&dmabuf->cq_event.cqe.rcqe_cmpl) == CQE_CODE_RECEIVE_V1))
fcfi = bf_get(lpfc_rcqe_fcf_id_v1,
fcfi = bf_get(lpfc_rcqe_fcf_id,
&dmabuf->cq_event.cqe.rcqe_cmpl);
- vport = lpfc_fc_frame_to_vport(phba, fc_hdr, fcfi);
+ /* d_id this frame is directed to */
+ did = sli4_did_from_fc_hdr(fc_hdr);
+
+ vport = lpfc_fc_frame_to_vport(phba, fc_hdr, fcfi, did);
if (!vport) {
/* throw out the frame */
lpfc_in_buf_free(phba, &dmabuf->dbuf);
return;
}
- /* d_id this frame is directed to */
- did = sli4_did_from_fc_hdr(fc_hdr);
-
/* vport is registered unless we rcv a FLOGI directed to Fabric_DID */
if (!(vport->vpi_state & LPFC_VPI_REGISTERED) &&
(did != Fabric_DID)) {
lpfc_drain_txq(struct lpfc_hba *phba)
{
LIST_HEAD(completions);
- struct lpfc_sli_ring *pring = &phba->sli.ring[LPFC_ELS_RING];
+ struct lpfc_sli_ring *pring;
struct lpfc_iocbq *piocbq = NULL;
unsigned long iflags = 0;
char *fail_msg = NULL;
union lpfc_wqe *wqe = (union lpfc_wqe *) &wqe128;
uint32_t txq_cnt = 0;
+ pring = lpfc_phba_elsring(phba);
+
spin_lock_irqsave(&pring->ring_lock, iflags);
list_for_each_entry(piocbq, &pring->txq, list) {
txq_cnt++;
txq_cnt);
break;
}
- sglq = __lpfc_sli_get_sglq(phba, piocbq);
+ sglq = __lpfc_sli_get_els_sglq(phba, piocbq);
if (!sglq) {
__lpfc_sli_ringtx_put(phba, pring, piocbq);
spin_unlock_irqrestore(&pring->ring_lock, iflags);
return txq_cnt;
}
+
+/**
+ * lpfc_wqe_bpl2sgl - Convert the bpl/bde to a sgl.
+ * @phba: Pointer to HBA context object.
+ * @pwqe: Pointer to command WQE.
+ * @sglq: Pointer to the scatter gather queue object.
+ *
+ * This routine converts the bpl or bde that is in the WQE
+ * to a sgl list for the sli4 hardware. The physical address
+ * of the bpl/bde is converted back to a virtual address.
+ * If the WQE contains a BPL then the list of BDE's is
+ * converted to sli4_sge's. If the WQE contains a single
+ * BDE then it is converted to a single sli_sge.
+ * The WQE is still in cpu endianness so the contents of
+ * the bpl can be used without byte swapping.
+ *
+ * Returns valid XRI = Success, NO_XRI = Failure.
+ */
+static uint16_t
+lpfc_wqe_bpl2sgl(struct lpfc_hba *phba, struct lpfc_iocbq *pwqeq,
+ struct lpfc_sglq *sglq)
+{
+ uint16_t xritag = NO_XRI;
+ struct ulp_bde64 *bpl = NULL;
+ struct ulp_bde64 bde;
+ struct sli4_sge *sgl = NULL;
+ struct lpfc_dmabuf *dmabuf;
+ union lpfc_wqe *wqe;
+ int numBdes = 0;
+ int i = 0;
+ uint32_t offset = 0; /* accumulated offset in the sg request list */
+ int inbound = 0; /* number of sg reply entries inbound from firmware */
+ uint32_t cmd;
+
+ if (!pwqeq || !sglq)
+ return xritag;
+
+ sgl = (struct sli4_sge *)sglq->sgl;
+ wqe = &pwqeq->wqe;
+ pwqeq->iocb.ulpIoTag = pwqeq->iotag;
+
+ cmd = bf_get(wqe_cmnd, &wqe->generic.wqe_com);
+ if (cmd == CMD_XMIT_BLS_RSP64_WQE)
+ return sglq->sli4_xritag;
+ numBdes = pwqeq->rsvd2;
+ if (numBdes) {
+ /* The addrHigh and addrLow fields within the WQE
+ * have not been byteswapped yet so there is no
+ * need to swap them back.
+ */
+ if (pwqeq->context3)
+ dmabuf = (struct lpfc_dmabuf *)pwqeq->context3;
+ else
+ return xritag;
+
+ bpl = (struct ulp_bde64 *)dmabuf->virt;
+ if (!bpl)
+ return xritag;
+
+ for (i = 0; i < numBdes; i++) {
+ /* Should already be byte swapped. */
+ sgl->addr_hi = bpl->addrHigh;
+ sgl->addr_lo = bpl->addrLow;
+
+ sgl->word2 = le32_to_cpu(sgl->word2);
+ if ((i+1) == numBdes)
+ bf_set(lpfc_sli4_sge_last, sgl, 1);
+ else
+ bf_set(lpfc_sli4_sge_last, sgl, 0);
+ /* swap the size field back to the cpu so we
+ * can assign it to the sgl.
+ */
+ bde.tus.w = le32_to_cpu(bpl->tus.w);
+ sgl->sge_len = cpu_to_le32(bde.tus.f.bdeSize);
+ /* The offsets in the sgl need to be accumulated
+ * separately for the request and reply lists.
+ * The request is always first, the reply follows.
+ */
+ switch (cmd) {
+ case CMD_GEN_REQUEST64_WQE:
+ /* add up the reply sg entries */
+ if (bpl->tus.f.bdeFlags == BUFF_TYPE_BDE_64I)
+ inbound++;
+ /* first inbound? reset the offset */
+ if (inbound == 1)
+ offset = 0;
+ bf_set(lpfc_sli4_sge_offset, sgl, offset);
+ bf_set(lpfc_sli4_sge_type, sgl,
+ LPFC_SGE_TYPE_DATA);
+ offset += bde.tus.f.bdeSize;
+ break;
+ case CMD_FCP_TRSP64_WQE:
+ bf_set(lpfc_sli4_sge_offset, sgl, 0);
+ bf_set(lpfc_sli4_sge_type, sgl,
+ LPFC_SGE_TYPE_DATA);
+ break;
+ case CMD_FCP_TSEND64_WQE:
+ case CMD_FCP_TRECEIVE64_WQE:
+ bf_set(lpfc_sli4_sge_type, sgl,
+ bpl->tus.f.bdeFlags);
+ if (i < 3)
+ offset = 0;
+ else
+ offset += bde.tus.f.bdeSize;
+ bf_set(lpfc_sli4_sge_offset, sgl, offset);
+ break;
+ }
+ sgl->word2 = cpu_to_le32(sgl->word2);
+ bpl++;
+ sgl++;
+ }
+ } else if (wqe->gen_req.bde.tus.f.bdeFlags == BUFF_TYPE_BDE_64) {
+ /* The addrHigh and addrLow fields of the BDE have not
+ * been byteswapped yet so they need to be swapped
+ * before putting them in the sgl.
+ */
+ sgl->addr_hi = cpu_to_le32(wqe->gen_req.bde.addrHigh);
+ sgl->addr_lo = cpu_to_le32(wqe->gen_req.bde.addrLow);
+ sgl->word2 = le32_to_cpu(sgl->word2);
+ bf_set(lpfc_sli4_sge_last, sgl, 1);
+ sgl->word2 = cpu_to_le32(sgl->word2);
+ sgl->sge_len = cpu_to_le32(wqe->gen_req.bde.tus.f.bdeSize);
+ }
+ return sglq->sli4_xritag;
+}
+
+/**
+ * lpfc_sli4_issue_wqe - Issue an SLI4 Work Queue Entry (WQE)
+ * @phba: Pointer to HBA context object.
+ * @ring_number: Base sli ring number
+ * @pwqe: Pointer to command WQE.
+ **/
+int
+lpfc_sli4_issue_wqe(struct lpfc_hba *phba, uint32_t ring_number,
+ struct lpfc_iocbq *pwqe)
+{
+ union lpfc_wqe *wqe = &pwqe->wqe;
+ struct lpfc_nvmet_rcv_ctx *ctxp;
+ struct lpfc_queue *wq;
+ struct lpfc_sglq *sglq;
+ struct lpfc_sli_ring *pring;
+ unsigned long iflags;
+
+ /* NVME_LS and NVME_LS ABTS requests. */
+ if (pwqe->iocb_flag & LPFC_IO_NVME_LS) {
+ pring = phba->sli4_hba.nvmels_wq->pring;
+ spin_lock_irqsave(&pring->ring_lock, iflags);
+ sglq = __lpfc_sli_get_els_sglq(phba, pwqe);
+ if (!sglq) {
+ spin_unlock_irqrestore(&pring->ring_lock, iflags);
+ return WQE_BUSY;
+ }
+ pwqe->sli4_lxritag = sglq->sli4_lxritag;
+ pwqe->sli4_xritag = sglq->sli4_xritag;
+ if (lpfc_wqe_bpl2sgl(phba, pwqe, sglq) == NO_XRI) {
+ spin_unlock_irqrestore(&pring->ring_lock, iflags);
+ return WQE_ERROR;
+ }
+ bf_set(wqe_xri_tag, &pwqe->wqe.xmit_bls_rsp.wqe_com,
+ pwqe->sli4_xritag);
+ if (lpfc_sli4_wq_put(phba->sli4_hba.nvmels_wq, wqe)) {
+ spin_unlock_irqrestore(&pring->ring_lock, iflags);
+ return WQE_ERROR;
+ }
+ lpfc_sli_ringtxcmpl_put(phba, pring, pwqe);
+ spin_unlock_irqrestore(&pring->ring_lock, iflags);
+ return 0;
+ }
+
+ /* NVME_FCREQ and NVME_ABTS requests */
+ if (pwqe->iocb_flag & LPFC_IO_NVME) {
+ /* Get the IO distribution (hba_wqidx) for WQ assignment. */
+ pring = phba->sli4_hba.nvme_wq[pwqe->hba_wqidx]->pring;
+
+ spin_lock_irqsave(&pring->ring_lock, iflags);
+ wq = phba->sli4_hba.nvme_wq[pwqe->hba_wqidx];
+ bf_set(wqe_cqid, &wqe->generic.wqe_com,
+ phba->sli4_hba.nvme_cq[pwqe->hba_wqidx]->queue_id);
+ if (lpfc_sli4_wq_put(wq, wqe)) {
+ spin_unlock_irqrestore(&pring->ring_lock, iflags);
+ return WQE_ERROR;
+ }
+ lpfc_sli_ringtxcmpl_put(phba, pring, pwqe);
+ spin_unlock_irqrestore(&pring->ring_lock, iflags);
+ return 0;
+ }
+
+ /* NVMET requests */
+ if (pwqe->iocb_flag & LPFC_IO_NVMET) {
+ /* Get the IO distribution (hba_wqidx) for WQ assignment. */
+ pring = phba->sli4_hba.nvme_wq[pwqe->hba_wqidx]->pring;
+
+ spin_lock_irqsave(&pring->ring_lock, iflags);
+ ctxp = pwqe->context2;
+ sglq = ctxp->rqb_buffer->sglq;
+ if (pwqe->sli4_xritag == NO_XRI) {
+ pwqe->sli4_lxritag = sglq->sli4_lxritag;
+ pwqe->sli4_xritag = sglq->sli4_xritag;
+ }
+ bf_set(wqe_xri_tag, &pwqe->wqe.xmit_bls_rsp.wqe_com,
+ pwqe->sli4_xritag);
+ wq = phba->sli4_hba.nvme_wq[pwqe->hba_wqidx];
+ bf_set(wqe_cqid, &wqe->generic.wqe_com,
+ phba->sli4_hba.nvme_cq[pwqe->hba_wqidx]->queue_id);
+ if (lpfc_sli4_wq_put(wq, wqe)) {
+ spin_unlock_irqrestore(&pring->ring_lock, iflags);
+ return WQE_ERROR;
+ }
+ lpfc_sli_ringtxcmpl_put(phba, pring, pwqe);
+ spin_unlock_irqrestore(&pring->ring_lock, iflags);
+ return 0;
+ }
+ return WQE_ERROR;
+}
/*******************************************************************
* This file is part of the Emulex Linux Device Driver for *
* Fibre Channel Host Bus Adapters. *
+ * Copyright (C) 2017 Broadcom. All Rights Reserved. The term *
+ * “Broadcom” refers to Broadcom Limited and/or its subsidiaries. *
* Copyright (C) 2004-2016 Emulex. All rights reserved. *
* EMULEX and SLI are trademarks of Emulex. *
- * www.emulex.com *
+ * www.broadcom.com *
* *
* This program is free software; you can redistribute it and/or *
* modify it under the terms of version 2 of the GNU General *
uint16_t iotag; /* pre-assigned IO tag */
uint16_t sli4_lxritag; /* logical pre-assigned XRI. */
uint16_t sli4_xritag; /* pre-assigned XRI, (OXID) tag. */
+ uint16_t hba_wqidx; /* index to HBA work queue */
struct lpfc_cq_event cq_event;
+ struct lpfc_wcqe_complete wcqe_cmpl; /* WQE cmpl */
+ uint64_t isr_timestamp;
- IOCB_t iocb; /* IOCB cmd */
+ /* Be careful here */
+ union lpfc_wqe wqe; /* WQE cmd */
+ IOCB_t iocb; /* For IOCB cmd or if we want 128 byte WQE */
+
+ uint8_t rsvd2;
uint8_t priority; /* OAS priority */
uint8_t retry; /* retry counter for IOCB cmd - if needed */
uint32_t iocb_flag;
#define LPFC_IO_OAS 0x10000 /* OAS FCP IO */
#define LPFC_IO_FOF 0x20000 /* FOF FCP IO */
#define LPFC_IO_LOOPBACK 0x40000 /* Loopback IO */
+#define LPFC_PRLI_NVME_REQ 0x80000 /* This is an NVME PRLI. */
+#define LPFC_PRLI_FCP_REQ 0x100000 /* This is an NVME PRLI. */
+#define LPFC_IO_NVME 0x200000 /* NVME FCP command */
+#define LPFC_IO_NVME_LS 0x400000 /* NVME LS command */
+#define LPFC_IO_NVMET 0x800000 /* NVMET command */
uint32_t drvrTimeout; /* driver timeout in seconds */
- uint32_t fcp_wqidx; /* index to FCP work queue */
struct lpfc_vport *vport;/* virtual port pointer */
void *context1; /* caller context information */
void *context2; /* caller context information */
struct lpfc_node_rrq *rrq;
} context_un;
- void (*fabric_iocb_cmpl) (struct lpfc_hba *, struct lpfc_iocbq *,
+ void (*fabric_iocb_cmpl)(struct lpfc_hba *, struct lpfc_iocbq *,
struct lpfc_iocbq *);
- void (*wait_iocb_cmpl) (struct lpfc_hba *, struct lpfc_iocbq *,
+ void (*wait_iocb_cmpl)(struct lpfc_hba *, struct lpfc_iocbq *,
struct lpfc_iocbq *);
- void (*iocb_cmpl) (struct lpfc_hba *, struct lpfc_iocbq *,
+ void (*iocb_cmpl)(struct lpfc_hba *, struct lpfc_iocbq *,
struct lpfc_iocbq *);
+ void (*wqe_cmpl)(struct lpfc_hba *, struct lpfc_iocbq *,
+ struct lpfc_wcqe_complete *);
};
#define SLI_IOCB_RET_IOCB 1 /* Return IOCB if cmd ring full */
#define IOCB_ERROR 2
#define IOCB_TIMEDOUT 3
+#define SLI_WQE_RET_WQE 1 /* Return WQE if cmd ring full */
+
+#define WQE_SUCCESS 0
+#define WQE_BUSY 1
+#define WQE_ERROR 2
+#define WQE_TIMEDOUT 3
+#define WQE_ABORTED 4
+
#define LPFC_MBX_WAKE 1
#define LPFC_MBX_IMED_UNREG 2
#define LPFC_BLOCK_MGMT_IO 0x800 /* Don't allow mgmt mbx or iocb cmds */
#define LPFC_MENLO_MAINT 0x1000 /* need for menl fw download */
#define LPFC_SLI_ASYNC_MBX_BLK 0x2000 /* Async mailbox is blocked */
+#define LPFC_SLI_SUPPRESS_RSP 0x4000 /* Suppress RSP feature is supported */
- struct lpfc_sli_ring *ring;
- int fcp_ring; /* ring used for FCP initiator commands */
- int next_ring;
-
- int extra_ring; /* extra ring used for other protocols */
+ struct lpfc_sli_ring *sli3_ring;
struct lpfc_sli_stat slistat; /* SLI statistical info */
struct list_head mboxq;
/*******************************************************************
* This file is part of the Emulex Linux Device Driver for *
* Fibre Channel Host Bus Adapters. *
+ * Copyright (C) 2017 Broadcom. All Rights Reserved. The term *
+ * “Broadcom” refers to Broadcom Limited and/or its subsidiaries. *
* Copyright (C) 2009-2016 Emulex. All rights reserved. *
* EMULEX and SLI are trademarks of Emulex. *
- * www.emulex.com *
+ * www.broadcom.com *
* *
* This program is free software; you can redistribute it and/or *
* modify it under the terms of version 2 of the GNU General *
#define LPFC_NEMBED_MBOX_SGL_CNT 254
/* Multi-queue arrangement for FCP EQ/CQ/WQ tuples */
-#define LPFC_FCP_IO_CHAN_DEF 4
-#define LPFC_FCP_IO_CHAN_MIN 1
-#define LPFC_FCP_IO_CHAN_MAX 16
+#define LPFC_HBA_IO_CHAN_MIN 0
+#define LPFC_HBA_IO_CHAN_MAX 32
+#define LPFC_FCP_IO_CHAN_DEF 4
+#define LPFC_NVME_IO_CHAN_DEF 0
/* Number of channels used for Flash Optimized Fabric (FOF) operations */
LPFC_MBOX,
LPFC_FCP,
LPFC_ELS,
+ LPFC_NVME,
+ LPFC_NVMET,
+ LPFC_NVME_LS,
LPFC_USOL
};
struct lpfc_rqe *rqe;
};
+/* RQ buffer list */
+struct lpfc_rqb {
+ uint16_t entry_count; /* Current number of RQ slots */
+ uint16_t buffer_count; /* Current number of buffers posted */
+ struct list_head rqb_buffer_list; /* buffers assigned to this HBQ */
+ /* Callback for HBQ buffer allocation */
+ struct rqb_dmabuf *(*rqb_alloc_buffer)(struct lpfc_hba *);
+ /* Callback for HBQ buffer free */
+ void (*rqb_free_buffer)(struct lpfc_hba *,
+ struct rqb_dmabuf *);
+};
+
struct lpfc_queue {
struct list_head list;
+ struct list_head wq_list;
enum lpfc_sli4_queue_type type;
enum lpfc_sli4_queue_subtype subtype;
struct lpfc_hba *phba;
struct list_head child_list;
+ struct list_head page_list;
+ struct list_head sgl_list;
uint32_t entry_count; /* Number of entries to support on the queue */
uint32_t entry_size; /* Size of each queue entry. */
uint32_t entry_repost; /* Count of entries before doorbell is rung */
#define LPFC_QUEUE_MIN_REPOST 8
uint32_t queue_id; /* Queue ID assigned by the hardware */
uint32_t assoc_qid; /* Queue ID associated with, for CQ/WQ/MQ */
- struct list_head page_list;
uint32_t page_count; /* Number of pages allocated for this queue */
uint32_t host_index; /* The host's index for putting or getting */
uint32_t hba_index; /* The last known hba index for get or put */
struct lpfc_sli_ring *pring; /* ptr to io ring associated with q */
+ struct lpfc_rqb *rqbp; /* ptr to RQ buffers */
+ uint16_t sgl_list_cnt;
uint16_t db_format;
#define LPFC_DB_RING_FORMAT 0x01
#define LPFC_DB_LIST_FORMAT 0x02
#define RQ_buf_trunc q_cnt_3
#define RQ_rcv_buf q_cnt_4
+ uint64_t isr_timestamp;
+ struct lpfc_queue *assoc_qp;
union sli4_qe qe[1]; /* array to index entries (must be last) */
};
#define LPFC_CQE_DEF_COUNT 1024
#define LPFC_WQE_DEF_COUNT 256
#define LPFC_WQE128_DEF_COUNT 128
+#define LPFC_WQE128_MAX_COUNT 256
#define LPFC_MQE_DEF_COUNT 16
#define LPFC_RQE_DEF_COUNT 512
struct lpfc_hba;
/* SLI4 HBA multi-fcp queue handler struct */
-struct lpfc_fcp_eq_hdl {
+struct lpfc_hba_eq_hdl {
uint32_t idx;
struct lpfc_hba *phba;
- atomic_t fcp_eq_in_use;
+ atomic_t hba_eq_in_use;
+ struct cpumask *cpumask;
+ /* CPU affinitsed to or 0xffffffff if multiple */
+ uint32_t cpu;
+#define LPFC_MULTI_CPU_AFFINITY 0xffffffff
};
/* Port Capabilities for SLI4 Parameters */
uint8_t wqsize;
#define LPFC_WQ_SZ64_SUPPORT 1
#define LPFC_WQ_SZ128_SUPPORT 2
+ uint8_t wqpcnt;
};
struct lpfc_iov {
uint8_t optic_state;
};
-#define LPFC_SLI4_HANDLER_CNT (LPFC_FCP_IO_CHAN_MAX+ \
+#define LPFC_SLI4_HANDLER_CNT (LPFC_HBA_IO_CHAN_MAX+ \
LPFC_FOF_IO_CHAN_NUM)
#define LPFC_SLI4_HANDLER_NAME_SZ 16
uint32_t ue_to_rp;
struct lpfc_register sli_intf;
struct lpfc_pc_sli4_params pc_sli4_params;
- struct msix_entry *msix_entries;
uint8_t handler_name[LPFC_SLI4_HANDLER_CNT][LPFC_SLI4_HANDLER_NAME_SZ];
- struct lpfc_fcp_eq_hdl *fcp_eq_hdl; /* FCP per-WQ handle */
+ struct lpfc_hba_eq_hdl *hba_eq_hdl; /* HBA per-WQ handle */
/* Pointers to the constructed SLI4 queues */
- struct lpfc_queue **hba_eq;/* Event queues for HBA */
- struct lpfc_queue **fcp_cq;/* Fast-path FCP compl queue */
- struct lpfc_queue **fcp_wq;/* Fast-path FCP work queue */
+ struct lpfc_queue **hba_eq; /* Event queues for HBA */
+ struct lpfc_queue **fcp_cq; /* Fast-path FCP compl queue */
+ struct lpfc_queue **nvme_cq; /* Fast-path NVME compl queue */
+ struct lpfc_queue **nvmet_cqset; /* Fast-path NVMET CQ Set queues */
+ struct lpfc_queue **nvmet_mrq_hdr; /* Fast-path NVMET hdr MRQs */
+ struct lpfc_queue **nvmet_mrq_data; /* Fast-path NVMET data MRQs */
+ struct lpfc_queue **fcp_wq; /* Fast-path FCP work queue */
+ struct lpfc_queue **nvme_wq; /* Fast-path NVME work queue */
uint16_t *fcp_cq_map;
+ uint16_t *nvme_cq_map;
+ struct list_head lpfc_wq_list;
struct lpfc_queue *mbx_cq; /* Slow-path mailbox complete queue */
struct lpfc_queue *els_cq; /* Slow-path ELS response complete queue */
+ struct lpfc_queue *nvmels_cq; /* NVME LS complete queue */
struct lpfc_queue *mbx_wq; /* Slow-path MBOX work queue */
struct lpfc_queue *els_wq; /* Slow-path ELS work queue */
+ struct lpfc_queue *nvmels_wq; /* NVME LS work queue */
struct lpfc_queue *hdr_rq; /* Slow-path Header Receive queue */
struct lpfc_queue *dat_rq; /* Slow-path Data Receive queue */
+ struct lpfc_name wwnn;
+ struct lpfc_name wwpn;
+
uint32_t fw_func_mode; /* FW function protocol mode */
uint32_t ulp0_mode; /* ULP0 protocol mode */
uint32_t ulp1_mode; /* ULP1 protocol mode */
uint16_t rpi_hdrs_in_use; /* must post rpi hdrs if set. */
uint16_t next_xri; /* last_xri - max_cfg_param.xri_base = used */
uint16_t next_rpi;
+ uint16_t nvme_xri_max;
+ uint16_t nvme_xri_cnt;
+ uint16_t nvme_xri_start;
uint16_t scsi_xri_max;
uint16_t scsi_xri_cnt;
- uint16_t els_xri_cnt;
uint16_t scsi_xri_start;
- struct list_head lpfc_free_sgl_list;
- struct list_head lpfc_sgl_list;
+ uint16_t els_xri_cnt;
+ uint16_t nvmet_xri_cnt;
+ struct list_head lpfc_els_sgl_list;
struct list_head lpfc_abts_els_sgl_list;
+ struct list_head lpfc_nvmet_sgl_list;
+ struct list_head lpfc_abts_nvmet_sgl_list;
struct list_head lpfc_abts_scsi_buf_list;
+ struct list_head lpfc_abts_nvme_buf_list;
struct lpfc_sglq **lpfc_sglq_active_list;
struct list_head lpfc_rpi_hdr_list;
unsigned long *rpi_bmask;
#define LPFC_SLI4_PPNAME_NON 0
#define LPFC_SLI4_PPNAME_GET 1
struct lpfc_iov iov;
+ spinlock_t abts_nvme_buf_list_lock; /* list of aborted SCSI IOs */
spinlock_t abts_scsi_buf_list_lock; /* list of aborted SCSI IOs */
- spinlock_t abts_sgl_list_lock; /* list of aborted els IOs */
+ spinlock_t sgl_list_lock; /* list of aborted els IOs */
+ spinlock_t nvmet_io_lock;
uint32_t physical_port;
/* CPU to vector mapping information */
uint16_t num_online_cpu;
uint16_t num_present_cpu;
uint16_t curr_disp_cpu;
+
+ uint16_t nvmet_mrq_post_idx;
};
enum lpfc_sge_type {
GEN_BUFF_TYPE,
- SCSI_BUFF_TYPE
+ SCSI_BUFF_TYPE,
+ NVMET_BUFF_TYPE
};
enum lpfc_sgl_state {
uint32_t);
void lpfc_sli4_queue_free(struct lpfc_queue *);
int lpfc_eq_create(struct lpfc_hba *, struct lpfc_queue *, uint32_t);
-int lpfc_modify_fcp_eq_delay(struct lpfc_hba *, uint32_t);
+int lpfc_modify_hba_eq_delay(struct lpfc_hba *phba, uint32_t startq);
int lpfc_cq_create(struct lpfc_hba *, struct lpfc_queue *,
struct lpfc_queue *, uint32_t, uint32_t);
+int lpfc_cq_create_set(struct lpfc_hba *phba, struct lpfc_queue **cqp,
+ struct lpfc_queue **eqp, uint32_t type,
+ uint32_t subtype);
int32_t lpfc_mq_create(struct lpfc_hba *, struct lpfc_queue *,
struct lpfc_queue *, uint32_t);
int lpfc_wq_create(struct lpfc_hba *, struct lpfc_queue *,
struct lpfc_queue *, uint32_t);
int lpfc_rq_create(struct lpfc_hba *, struct lpfc_queue *,
struct lpfc_queue *, struct lpfc_queue *, uint32_t);
+int lpfc_mrq_create(struct lpfc_hba *phba, struct lpfc_queue **hrqp,
+ struct lpfc_queue **drqp, struct lpfc_queue **cqp,
+ uint32_t subtype);
void lpfc_rq_adjust_repost(struct lpfc_hba *, struct lpfc_queue *, int);
int lpfc_eq_destroy(struct lpfc_hba *, struct lpfc_queue *);
int lpfc_cq_destroy(struct lpfc_hba *, struct lpfc_queue *);
void lpfc_sli4_queue_unset(struct lpfc_hba *);
int lpfc_sli4_post_sgl(struct lpfc_hba *, dma_addr_t, dma_addr_t, uint16_t);
int lpfc_sli4_repost_scsi_sgl_list(struct lpfc_hba *);
+int lpfc_repost_nvme_sgl_list(struct lpfc_hba *phba);
uint16_t lpfc_sli4_next_xritag(struct lpfc_hba *);
void lpfc_sli4_free_xri(struct lpfc_hba *, int);
int lpfc_sli4_post_async_mbox(struct lpfc_hba *);
int lpfc_sli4_add_fcf_record(struct lpfc_hba *, struct fcf_record *);
void lpfc_sli_remove_dflt_fcf(struct lpfc_hba *);
int lpfc_sli4_get_els_iocb_cnt(struct lpfc_hba *);
+int lpfc_sli4_get_iocb_cnt(struct lpfc_hba *phba);
int lpfc_sli4_init_vpi(struct lpfc_vport *);
uint32_t lpfc_sli4_cq_release(struct lpfc_queue *, bool);
uint32_t lpfc_sli4_eq_release(struct lpfc_queue *, bool);
/*******************************************************************
* This file is part of the Emulex Linux Device Driver for *
* Fibre Channel Host Bus Adapters. *
+ * Copyright (C) 2017 Broadcom. All Rights Reserved. The term *
+ * “Broadcom” refers to Broadcom Limited and/or its subsidiaries. *
* Copyright (C) 2004-2016 Emulex. All rights reserved. *
* EMULEX and SLI are trademarks of Emulex. *
- * www.emulex.com *
+ * www.broadcom.com *
* *
* This program is free software; you can redistribute it and/or *
* modify it under the terms of version 2 of the GNU General *
* included with this package. *
*******************************************************************/
-#define LPFC_DRIVER_VERSION "11.2.0.4"
+#define LPFC_DRIVER_VERSION "11.2.0.7"
#define LPFC_DRIVER_NAME "lpfc"
/* Used for SLI 2/3 */
#define LPFC_MODULE_DESC "Emulex LightPulse Fibre Channel SCSI driver " \
LPFC_DRIVER_VERSION
-#define LPFC_COPYRIGHT "Copyright(c) 2004-2016 Emulex. All rights reserved."
+#define LPFC_COPYRIGHT "Copyright (C) 2017 Broadcom. All Rights Reserved. " \
+ "The term \"Broadcom\" refers to Broadcom Limited " \
+ "and/or its subsidiaries."
/*******************************************************************
* This file is part of the Emulex Linux Device Driver for *
* Fibre Channel Host Bus Adapters. *
+ * Copyright (C) 2017 Broadcom. All Rights Reserved. The term *
+ * “Broadcom” refers to Broadcom Limited and/or its subsidiaries. *
* Copyright (C) 2004-2016 Emulex. All rights reserved. *
* EMULEX and SLI are trademarks of Emulex. *
- * www.emulex.com *
+ * www.broadcom.com *
* Portions Copyright (C) 2004-2005 Christoph Hellwig *
* *
* This program is free software; you can redistribute it and/or *
#include <linux/pci.h>
#include <linux/slab.h>
#include <linux/spinlock.h>
+#include <linux/sched/signal.h>
#include <scsi/scsi.h>
#include <scsi/scsi_device.h>
#include <scsi/scsi_host.h>
#include <scsi/scsi_transport_fc.h>
+
#include "lpfc_hw4.h"
#include "lpfc_hw.h"
#include "lpfc_sli.h"
vport->fdmi_port_mask = phba->pport->fdmi_port_mask;
}
+ if ((phba->nvmet_support == 0) &&
+ ((phba->cfg_enable_fc4_type == LPFC_ENABLE_BOTH) ||
+ (phba->cfg_enable_fc4_type == LPFC_ENABLE_NVME))) {
+ /* Create NVME binding with nvme_fc_transport. This
+ * ensures the vport is initialized.
+ */
+ rc = lpfc_nvme_create_localport(vport);
+ if (rc) {
+ lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+ "6003 %s status x%x\n",
+ "NVME registration failed, ",
+ rc);
+ goto error_out;
+ }
+ }
+
/*
* In SLI4, the vpi must be activated before it can be used
* by the port.
/*******************************************************************
* This file is part of the Emulex Linux Device Driver for *
* Fibre Channel Host Bus Adapters. *
+ * Copyright (C) 2017 Broadcom. All Rights Reserved. The term *
+ * “Broadcom” refers to Broadcom Limited and/or its subsidiaries. *
* Copyright (C) 2004-2006 Emulex. All rights reserved. *
* EMULEX and SLI are trademarks of Emulex. *
- * www.emulex.com *
+ * www.broadcom.com *
* Portions Copyright (C) 2004-2005 Christoph Hellwig *
* *
* This program is free software; you can redistribute it and/or *
/* TMs are on msix_index == 0 */
if (reply_q->msix_index == 0)
continue;
- synchronize_irq(reply_q->vector);
+ synchronize_irq(pci_irq_vector(ioc->pdev, reply_q->msix_index));
}
}
list_for_each_entry_safe(reply_q, next, &ioc->reply_queue_list, list) {
list_del(&reply_q->list);
- if (smp_affinity_enable) {
- irq_set_affinity_hint(reply_q->vector, NULL);
- free_cpumask_var(reply_q->affinity_hint);
- }
- free_irq(reply_q->vector, reply_q);
+ free_irq(pci_irq_vector(ioc->pdev, reply_q->msix_index),
+ reply_q);
kfree(reply_q);
}
}
* _base_request_irq - request irq
* @ioc: per adapter object
* @index: msix index into vector table
- * @vector: irq vector
*
* Inserting respective reply_queue into the list.
*/
static int
-_base_request_irq(struct MPT3SAS_ADAPTER *ioc, u8 index, u32 vector)
+_base_request_irq(struct MPT3SAS_ADAPTER *ioc, u8 index)
{
+ struct pci_dev *pdev = ioc->pdev;
struct adapter_reply_queue *reply_q;
int r;
}
reply_q->ioc = ioc;
reply_q->msix_index = index;
- reply_q->vector = vector;
-
- if (smp_affinity_enable) {
- if (!zalloc_cpumask_var(&reply_q->affinity_hint, GFP_KERNEL)) {
- kfree(reply_q);
- return -ENOMEM;
- }
- }
atomic_set(&reply_q->busy, 0);
if (ioc->msix_enable)
else
snprintf(reply_q->name, MPT_NAME_LENGTH, "%s%d",
ioc->driver_name, ioc->id);
- r = request_irq(vector, _base_interrupt, IRQF_SHARED, reply_q->name,
- reply_q);
+ r = request_irq(pci_irq_vector(pdev, index), _base_interrupt,
+ IRQF_SHARED, reply_q->name, reply_q);
if (r) {
pr_err(MPT3SAS_FMT "unable to allocate interrupt %d!\n",
- reply_q->name, vector);
- free_cpumask_var(reply_q->affinity_hint);
+ reply_q->name, pci_irq_vector(pdev, index));
kfree(reply_q);
return -EBUSY;
}
if (!nr_msix)
return;
+ if (smp_affinity_enable) {
+ list_for_each_entry(reply_q, &ioc->reply_queue_list, list) {
+ const cpumask_t *mask = pci_irq_get_affinity(ioc->pdev,
+ reply_q->msix_index);
+ if (!mask) {
+ pr_warn(MPT3SAS_FMT "no affinity for msi %x\n",
+ ioc->name, reply_q->msix_index);
+ continue;
+ }
+
+ for_each_cpu(cpu, mask)
+ ioc->cpu_msix_table[cpu] = reply_q->msix_index;
+ }
+ return;
+ }
cpu = cpumask_first(cpu_online_mask);
list_for_each_entry(reply_q, &ioc->reply_queue_list, list) {
group++;
for (i = 0 ; i < group ; i++) {
- ioc->cpu_msix_table[cpu] = index;
- if (smp_affinity_enable)
- cpumask_or(reply_q->affinity_hint,
- reply_q->affinity_hint, get_cpu_mask(cpu));
+ ioc->cpu_msix_table[cpu] = reply_q->msix_index;
cpu = cpumask_next(cpu, cpu_online_mask);
}
- if (smp_affinity_enable)
- if (irq_set_affinity_hint(reply_q->vector,
- reply_q->affinity_hint))
- dinitprintk(ioc, pr_info(MPT3SAS_FMT
- "Err setting affinity hint to irq vector %d\n",
- ioc->name, reply_q->vector));
index++;
}
}
static int
_base_enable_msix(struct MPT3SAS_ADAPTER *ioc)
{
- struct msix_entry *entries, *a;
int r;
int i, local_max_msix_vectors;
u8 try_msix = 0;
+ unsigned int irq_flags = PCI_IRQ_MSIX;
if (msix_disable == -1 || msix_disable == 0)
try_msix = 1;
goto try_ioapic;
ioc->reply_queue_count = min_t(int, ioc->cpu_count,
- ioc->msix_vector_count);
+ ioc->msix_vector_count);
printk(MPT3SAS_FMT "MSI-X vectors supported: %d, no of cores"
": %d, max_msix_vectors: %d\n", ioc->name, ioc->msix_vector_count,
else
local_max_msix_vectors = max_msix_vectors;
- if (local_max_msix_vectors > 0) {
+ if (local_max_msix_vectors > 0)
ioc->reply_queue_count = min_t(int, local_max_msix_vectors,
ioc->reply_queue_count);
- ioc->msix_vector_count = ioc->reply_queue_count;
- } else if (local_max_msix_vectors == 0)
+ else if (local_max_msix_vectors == 0)
goto try_ioapic;
if (ioc->msix_vector_count < ioc->cpu_count)
smp_affinity_enable = 0;
- entries = kcalloc(ioc->reply_queue_count, sizeof(struct msix_entry),
- GFP_KERNEL);
- if (!entries) {
- dfailprintk(ioc, pr_info(MPT3SAS_FMT
- "kcalloc failed @ at %s:%d/%s() !!!\n",
- ioc->name, __FILE__, __LINE__, __func__));
- goto try_ioapic;
- }
+ if (smp_affinity_enable)
+ irq_flags |= PCI_IRQ_AFFINITY;
- for (i = 0, a = entries; i < ioc->reply_queue_count; i++, a++)
- a->entry = i;
-
- r = pci_enable_msix_exact(ioc->pdev, entries, ioc->reply_queue_count);
- if (r) {
+ r = pci_alloc_irq_vectors(ioc->pdev, 1, ioc->reply_queue_count,
+ irq_flags);
+ if (r < 0) {
dfailprintk(ioc, pr_info(MPT3SAS_FMT
- "pci_enable_msix_exact failed (r=%d) !!!\n",
+ "pci_alloc_irq_vectors failed (r=%d) !!!\n",
ioc->name, r));
- kfree(entries);
goto try_ioapic;
}
ioc->msix_enable = 1;
- for (i = 0, a = entries; i < ioc->reply_queue_count; i++, a++) {
- r = _base_request_irq(ioc, i, a->vector);
+ ioc->reply_queue_count = r;
+ for (i = 0; i < ioc->reply_queue_count; i++) {
+ r = _base_request_irq(ioc, i);
if (r) {
_base_free_irq(ioc);
_base_disable_msix(ioc);
- kfree(entries);
goto try_ioapic;
}
}
- kfree(entries);
return 0;
/* failback to io_apic interrupt routing */
try_ioapic:
ioc->reply_queue_count = 1;
- r = _base_request_irq(ioc, 0, ioc->pdev->irq);
+ r = pci_alloc_irq_vectors(ioc->pdev, 1, 1, PCI_IRQ_LEGACY);
+ if (r < 0) {
+ dfailprintk(ioc, pr_info(MPT3SAS_FMT
+ "pci_alloc_irq_vector(legacy) failed (r=%d) !!!\n",
+ ioc->name, r));
+ } else
+ r = _base_request_irq(ioc, 0);
return r;
}
list_for_each_entry(reply_q, &ioc->reply_queue_list, list)
pr_info(MPT3SAS_FMT "%s: IRQ %d\n",
reply_q->name, ((ioc->msix_enable) ? "PCI-MSI-X enabled" :
- "IO-APIC enabled"), reply_q->vector);
+ "IO-APIC enabled"),
+ pci_irq_vector(ioc->pdev, reply_q->msix_index));
pr_info(MPT3SAS_FMT "iomem(0x%016llx), mapped(0x%p), size(%d)\n",
ioc->name, (unsigned long long)chip_phys, ioc->chip, memap_sz);
sizeof(resource_size_t *), GFP_KERNEL);
if (!ioc->reply_post_host_index) {
dfailprintk(ioc, pr_info(MPT3SAS_FMT "allocation "
- "for cpu_msix_table failed!!!\n", ioc->name));
+ "for reply_post_host_index failed!!!\n",
+ ioc->name));
r = -ENOMEM;
goto out_free_resources;
}
struct adapter_reply_queue {
struct MPT3SAS_ADAPTER *ioc;
u8 msix_index;
- unsigned int vector;
u32 reply_post_host_index;
Mpi2ReplyDescriptorsUnion_t *reply_post_free;
char name[MPT_NAME_LENGTH];
atomic_t busy;
- cpumask_var_t affinity_hint;
struct list_head list;
};
static int __detect_osd(struct osd_uld_device *oud)
{
struct scsi_device *scsi_device = oud->od.scsi_device;
+ struct scsi_sense_hdr sense_hdr;
char caps[OSD_CAP_LEN];
int error;
*/
OSD_DEBUG("start scsi_test_unit_ready %p %p %p\n",
oud, scsi_device, scsi_device->request_queue);
- error = scsi_test_unit_ready(scsi_device, 10*HZ, 5, NULL);
+ error = scsi_test_unit_ready(scsi_device, 10*HZ, 5, &sense_hdr);
if (error)
OSD_ERR("warning: scsi_test_unit_ready failed\n");
#include <linux/fs.h>
#include <linux/kernel.h>
-#include <linux/sched.h>
+#include <linux/sched/signal.h>
#include <linux/proc_fs.h>
#include <linux/mm.h>
#include <linux/slab.h>
--- /dev/null
+config QEDF
+ tristate "QLogic QEDF 25/40/100Gb FCoE Initiator Driver Support"
+ depends on PCI && SCSI
+ depends on QED
+ depends on LIBFC
+ depends on LIBFCOE
+ select QED_LL2
+ select QED_FCOE
+ ---help---
+ This driver supports FCoE offload for the QLogic FastLinQ
+ 41000 Series Converged Network Adapters.
--- /dev/null
+obj-$(CONFIG_QEDF) := qedf.o
+qedf-y = qedf_dbg.o qedf_main.o qedf_io.o qedf_fip.o \
+ qedf_attr.o qedf_els.o
+
+qedf-$(CONFIG_DEBUG_FS) += qedf_debugfs.o
--- /dev/null
+/*
+ * QLogic FCoE Offload Driver
+ * Copyright (c) 2016 Cavium Inc.
+ *
+ * This software is available under the terms of the GNU General Public License
+ * (GPL) Version 2, available from the file COPYING in the main directory of
+ * this source tree.
+ */
+#ifndef _QEDFC_H_
+#define _QEDFC_H_
+
+#include <scsi/libfcoe.h>
+#include <scsi/libfc.h>
+#include <scsi/fc/fc_fip.h>
+#include <scsi/fc/fc_fc2.h>
+#include <scsi/scsi_tcq.h>
+#include <scsi/fc_encode.h>
+#include <linux/version.h>
+
+
+/* qedf_hsi.h needs to before included any qed includes */
+#include "qedf_hsi.h"
+
+#include <linux/qed/qed_if.h>
+#include <linux/qed/qed_fcoe_if.h>
+#include <linux/qed/qed_ll2_if.h>
+#include "qedf_version.h"
+#include "qedf_dbg.h"
+
+/* Helpers to extract upper and lower 32-bits of pointer */
+#define U64_HI(val) ((u32)(((u64)(val)) >> 32))
+#define U64_LO(val) ((u32)(((u64)(val)) & 0xffffffff))
+
+#define QEDF_DESCR "QLogic FCoE Offload Driver"
+#define QEDF_MODULE_NAME "qedf"
+
+#define QEDF_MIN_XID 0
+#define QEDF_MAX_SCSI_XID (NUM_TASKS_PER_CONNECTION - 1)
+#define QEDF_MAX_ELS_XID 4095
+#define QEDF_FLOGI_RETRY_CNT 3
+#define QEDF_RPORT_RETRY_CNT 255
+#define QEDF_MAX_SESSIONS 1024
+#define QEDF_MAX_PAYLOAD 2048
+#define QEDF_MAX_BDS_PER_CMD 256
+#define QEDF_MAX_BD_LEN 0xffff
+#define QEDF_BD_SPLIT_SZ 0x1000
+#define QEDF_PAGE_SIZE 4096
+#define QED_HW_DMA_BOUNDARY 0xfff
+#define QEDF_MAX_SGLEN_FOR_CACHESGL ((1U << 16) - 1)
+#define QEDF_MFS (QEDF_MAX_PAYLOAD + \
+ sizeof(struct fc_frame_header))
+#define QEDF_MAX_NPIV 64
+#define QEDF_TM_TIMEOUT 10
+#define QEDF_ABORT_TIMEOUT 10
+#define QEDF_CLEANUP_TIMEOUT 10
+#define QEDF_MAX_CDB_LEN 16
+
+#define UPSTREAM_REMOVE 1
+#define UPSTREAM_KEEP 1
+
+struct qedf_mp_req {
+ uint8_t tm_flags;
+
+ uint32_t req_len;
+ void *req_buf;
+ dma_addr_t req_buf_dma;
+ struct fcoe_sge *mp_req_bd;
+ dma_addr_t mp_req_bd_dma;
+ struct fc_frame_header req_fc_hdr;
+
+ uint32_t resp_len;
+ void *resp_buf;
+ dma_addr_t resp_buf_dma;
+ struct fcoe_sge *mp_resp_bd;
+ dma_addr_t mp_resp_bd_dma;
+ struct fc_frame_header resp_fc_hdr;
+};
+
+struct qedf_els_cb_arg {
+ struct qedf_ioreq *aborted_io_req;
+ struct qedf_ioreq *io_req;
+ u8 op; /* Used to keep track of ELS op */
+ uint16_t l2_oxid;
+ u32 offset; /* Used for sequence cleanup */
+ u8 r_ctl; /* Used for sequence cleanup */
+};
+
+enum qedf_ioreq_event {
+ QEDF_IOREQ_EV_ABORT_SUCCESS,
+ QEDF_IOREQ_EV_ABORT_FAILED,
+ QEDF_IOREQ_EV_SEND_RRQ,
+ QEDF_IOREQ_EV_ELS_TMO,
+ QEDF_IOREQ_EV_ELS_ERR_DETECT,
+ QEDF_IOREQ_EV_ELS_FLUSH,
+ QEDF_IOREQ_EV_CLEANUP_SUCCESS,
+ QEDF_IOREQ_EV_CLEANUP_FAILED,
+};
+
+#define FC_GOOD 0
+#define FCOE_FCP_RSP_FLAGS_FCP_RESID_OVER (0x1<<2)
+#define FCOE_FCP_RSP_FLAGS_FCP_RESID_UNDER (0x1<<3)
+#define CMD_SCSI_STATUS(Cmnd) ((Cmnd)->SCp.Status)
+#define FCOE_FCP_RSP_FLAGS_FCP_RSP_LEN_VALID (0x1<<0)
+#define FCOE_FCP_RSP_FLAGS_FCP_SNS_LEN_VALID (0x1<<1)
+struct qedf_ioreq {
+ struct list_head link;
+ uint16_t xid;
+ struct scsi_cmnd *sc_cmd;
+ bool use_slowpath; /* Use slow SGL for this I/O */
+#define QEDF_SCSI_CMD 1
+#define QEDF_TASK_MGMT_CMD 2
+#define QEDF_ABTS 3
+#define QEDF_ELS 4
+#define QEDF_CLEANUP 5
+#define QEDF_SEQ_CLEANUP 6
+ u8 cmd_type;
+#define QEDF_CMD_OUTSTANDING 0x0
+#define QEDF_CMD_IN_ABORT 0x1
+#define QEDF_CMD_IN_CLEANUP 0x2
+#define QEDF_CMD_SRR_SENT 0x3
+ u8 io_req_flags;
+ struct qedf_rport *fcport;
+ unsigned long flags;
+ enum qedf_ioreq_event event;
+ size_t data_xfer_len;
+ struct kref refcount;
+ struct qedf_cmd_mgr *cmd_mgr;
+ struct io_bdt *bd_tbl;
+ struct delayed_work timeout_work;
+ struct completion tm_done;
+ struct completion abts_done;
+ struct fcoe_task_context *task;
+ int idx;
+/*
+ * Need to allocate enough room for both sense data and FCP response data
+ * which has a max length of 8 bytes according to spec.
+ */
+#define QEDF_SCSI_SENSE_BUFFERSIZE (SCSI_SENSE_BUFFERSIZE + 8)
+ uint8_t *sense_buffer;
+ dma_addr_t sense_buffer_dma;
+ u32 fcp_resid;
+ u32 fcp_rsp_len;
+ u32 fcp_sns_len;
+ u8 cdb_status;
+ u8 fcp_status;
+ u8 fcp_rsp_code;
+ u8 scsi_comp_flags;
+#define QEDF_MAX_REUSE 0xfff
+ u16 reuse_count;
+ struct qedf_mp_req mp_req;
+ void (*cb_func)(struct qedf_els_cb_arg *cb_arg);
+ struct qedf_els_cb_arg *cb_arg;
+ int fp_idx;
+ unsigned int cpu;
+ unsigned int int_cpu;
+#define QEDF_IOREQ_SLOW_SGE 0
+#define QEDF_IOREQ_SINGLE_SGE 1
+#define QEDF_IOREQ_FAST_SGE 2
+ u8 sge_type;
+ struct delayed_work rrq_work;
+
+ /* Used for sequence level recovery; i.e. REC/SRR */
+ uint32_t rx_buf_off;
+ uint32_t tx_buf_off;
+ uint32_t rx_id;
+ uint32_t task_retry_identifier;
+
+ /*
+ * Used to tell if we need to return a SCSI command
+ * during some form of error processing.
+ */
+ bool return_scsi_cmd_on_abts;
+};
+
+extern struct workqueue_struct *qedf_io_wq;
+
+struct qedf_rport {
+ spinlock_t rport_lock;
+#define QEDF_RPORT_SESSION_READY 1
+#define QEDF_RPORT_UPLOADING_CONNECTION 2
+ unsigned long flags;
+ unsigned long retry_delay_timestamp;
+ struct fc_rport *rport;
+ struct fc_rport_priv *rdata;
+ struct qedf_ctx *qedf;
+ u32 handle; /* Handle from qed */
+ u32 fw_cid; /* fw_cid from qed */
+ void __iomem *p_doorbell;
+ /* Send queue management */
+ atomic_t free_sqes;
+ atomic_t num_active_ios;
+ struct fcoe_wqe *sq;
+ dma_addr_t sq_dma;
+ u16 sq_prod_idx;
+ u16 fw_sq_prod_idx;
+ u16 sq_con_idx;
+ u32 sq_mem_size;
+ void *sq_pbl;
+ dma_addr_t sq_pbl_dma;
+ u32 sq_pbl_size;
+ u32 sid;
+#define QEDF_RPORT_TYPE_DISK 1
+#define QEDF_RPORT_TYPE_TAPE 2
+ uint dev_type; /* Disk or tape */
+ struct list_head peers;
+};
+
+/* Used to contain LL2 skb's in ll2_skb_list */
+struct qedf_skb_work {
+ struct work_struct work;
+ struct sk_buff *skb;
+ struct qedf_ctx *qedf;
+};
+
+struct qedf_fastpath {
+#define QEDF_SB_ID_NULL 0xffff
+ u16 sb_id;
+ struct qed_sb_info *sb_info;
+ struct qedf_ctx *qedf;
+ /* Keep track of number of completions on this fastpath */
+ unsigned long completions;
+ uint32_t cq_num_entries;
+};
+
+/* Used to pass fastpath information needed to process CQEs */
+struct qedf_io_work {
+ struct work_struct work;
+ struct fcoe_cqe cqe;
+ struct qedf_ctx *qedf;
+ struct fc_frame *fp;
+};
+
+struct qedf_glbl_q_params {
+ u64 hw_p_cq; /* Completion queue PBL */
+ u64 hw_p_rq; /* Request queue PBL */
+ u64 hw_p_cmdq; /* Command queue PBL */
+};
+
+struct global_queue {
+ struct fcoe_cqe *cq;
+ dma_addr_t cq_dma;
+ u32 cq_mem_size;
+ u32 cq_cons_idx; /* Completion queue consumer index */
+ u32 cq_prod_idx;
+
+ void *cq_pbl;
+ dma_addr_t cq_pbl_dma;
+ u32 cq_pbl_size;
+};
+
+/* I/O tracing entry */
+#define QEDF_IO_TRACE_SIZE 2048
+struct qedf_io_log {
+#define QEDF_IO_TRACE_REQ 0
+#define QEDF_IO_TRACE_RSP 1
+ uint8_t direction;
+ uint16_t task_id;
+ uint32_t port_id; /* Remote port fabric ID */
+ int lun;
+ char op; /* SCSI CDB */
+ uint8_t lba[4];
+ unsigned int bufflen; /* SCSI buffer length */
+ unsigned int sg_count; /* Number of SG elements */
+ int result; /* Result passed back to mid-layer */
+ unsigned long jiffies; /* Time stamp when I/O logged */
+ int refcount; /* Reference count for task id */
+ unsigned int req_cpu; /* CPU that the task is queued on */
+ unsigned int int_cpu; /* Interrupt CPU that the task is received on */
+ unsigned int rsp_cpu; /* CPU that task is returned on */
+ u8 sge_type; /* Did we take the slow, single or fast SGE path */
+};
+
+/* Number of entries in BDQ */
+#define QEDF_BDQ_SIZE 256
+#define QEDF_BDQ_BUF_SIZE 2072
+
+/* DMA coherent buffers for BDQ */
+struct qedf_bdq_buf {
+ void *buf_addr;
+ dma_addr_t buf_dma;
+};
+
+/* Main adapter struct */
+struct qedf_ctx {
+ struct qedf_dbg_ctx dbg_ctx;
+ struct fcoe_ctlr ctlr;
+ struct fc_lport *lport;
+ u8 data_src_addr[ETH_ALEN];
+#define QEDF_LINK_DOWN 0
+#define QEDF_LINK_UP 1
+ atomic_t link_state;
+#define QEDF_DCBX_PENDING 0
+#define QEDF_DCBX_DONE 1
+ atomic_t dcbx;
+ uint16_t max_scsi_xid;
+ uint16_t max_els_xid;
+#define QEDF_NULL_VLAN_ID -1
+#define QEDF_FALLBACK_VLAN 1002
+#define QEDF_DEFAULT_PRIO 3
+ int vlan_id;
+ uint vlan_hw_insert:1;
+ struct qed_dev *cdev;
+ struct qed_dev_fcoe_info dev_info;
+ struct qed_int_info int_info;
+ uint16_t last_command;
+ spinlock_t hba_lock;
+ struct pci_dev *pdev;
+ u64 wwnn;
+ u64 wwpn;
+ u8 __aligned(16) mac[ETH_ALEN];
+ struct list_head fcports;
+ atomic_t num_offloads;
+ unsigned int curr_conn_id;
+ struct workqueue_struct *ll2_recv_wq;
+ struct workqueue_struct *link_update_wq;
+ struct delayed_work link_update;
+ struct delayed_work link_recovery;
+ struct completion flogi_compl;
+ struct completion fipvlan_compl;
+
+ /*
+ * Used to tell if we're in the window where we are waiting for
+ * the link to come back up before informting fcoe that the link is
+ * done.
+ */
+ atomic_t link_down_tmo_valid;
+#define QEDF_TIMER_INTERVAL (1 * HZ)
+ struct timer_list timer; /* One second book keeping timer */
+#define QEDF_DRAIN_ACTIVE 1
+#define QEDF_LL2_STARTED 2
+#define QEDF_UNLOADING 3
+#define QEDF_GRCDUMP_CAPTURE 4
+#define QEDF_IN_RECOVERY 5
+#define QEDF_DBG_STOP_IO 6
+ unsigned long flags; /* Miscellaneous state flags */
+ int fipvlan_retries;
+ u8 num_queues;
+ struct global_queue **global_queues;
+ /* Pointer to array of queue structures */
+ struct qedf_glbl_q_params *p_cpuq;
+ /* Physical address of array of queue structures */
+ dma_addr_t hw_p_cpuq;
+
+ struct qedf_bdq_buf bdq[QEDF_BDQ_SIZE];
+ void *bdq_pbl;
+ dma_addr_t bdq_pbl_dma;
+ size_t bdq_pbl_mem_size;
+ void *bdq_pbl_list;
+ dma_addr_t bdq_pbl_list_dma;
+ u8 bdq_pbl_list_num_entries;
+ void __iomem *bdq_primary_prod;
+ void __iomem *bdq_secondary_prod;
+ uint16_t bdq_prod_idx;
+
+ /* Structure for holding all the fastpath for this qedf_ctx */
+ struct qedf_fastpath *fp_array;
+ struct qed_fcoe_tid tasks;
+ struct qedf_cmd_mgr *cmd_mgr;
+ /* Holds the PF parameters we pass to qed to start he FCoE function */
+ struct qed_pf_params pf_params;
+ /* Used to time middle path ELS and TM commands */
+ struct workqueue_struct *timer_work_queue;
+
+#define QEDF_IO_WORK_MIN 64
+ mempool_t *io_mempool;
+ struct workqueue_struct *dpc_wq;
+
+ u32 slow_sge_ios;
+ u32 fast_sge_ios;
+ u32 single_sge_ios;
+
+ uint8_t *grcdump;
+ uint32_t grcdump_size;
+
+ struct qedf_io_log io_trace_buf[QEDF_IO_TRACE_SIZE];
+ spinlock_t io_trace_lock;
+ uint16_t io_trace_idx;
+
+ bool stop_io_on_error;
+
+ u32 flogi_cnt;
+ u32 flogi_failed;
+
+ /* Used for fc statistics */
+ u64 input_requests;
+ u64 output_requests;
+ u64 control_requests;
+ u64 packet_aborts;
+ u64 alloc_failures;
+};
+
+struct io_bdt {
+ struct qedf_ioreq *io_req;
+ struct fcoe_sge *bd_tbl;
+ dma_addr_t bd_tbl_dma;
+ u16 bd_valid;
+};
+
+struct qedf_cmd_mgr {
+ struct qedf_ctx *qedf;
+ u16 idx;
+ struct io_bdt **io_bdt_pool;
+#define FCOE_PARAMS_NUM_TASKS 4096
+ struct qedf_ioreq cmds[FCOE_PARAMS_NUM_TASKS];
+ spinlock_t lock;
+ atomic_t free_list_cnt;
+};
+
+/* Stolen from qed_cxt_api.h and adapted for qed_fcoe_info
+ * Usage:
+ *
+ * void *ptr;
+ * ptr = qedf_get_task_mem(&qedf->tasks, 128);
+ */
+static inline void *qedf_get_task_mem(struct qed_fcoe_tid *info, u32 tid)
+{
+ return (void *)(info->blocks[tid / info->num_tids_per_block] +
+ (tid % info->num_tids_per_block) * info->size);
+}
+
+static inline void qedf_stop_all_io(struct qedf_ctx *qedf)
+{
+ set_bit(QEDF_DBG_STOP_IO, &qedf->flags);
+}
+
+/*
+ * Externs
+ */
+#define QEDF_DEFAULT_LOG_MASK 0x3CFB6
+extern const struct qed_fcoe_ops *qed_ops;
+extern uint qedf_dump_frames;
+extern uint qedf_io_tracing;
+extern uint qedf_stop_io_on_error;
+extern uint qedf_link_down_tmo;
+#define QEDF_RETRY_DELAY_MAX 20 /* 2 seconds */
+extern bool qedf_retry_delay;
+extern uint qedf_debug;
+
+extern struct qedf_cmd_mgr *qedf_cmd_mgr_alloc(struct qedf_ctx *qedf);
+extern void qedf_cmd_mgr_free(struct qedf_cmd_mgr *cmgr);
+extern int qedf_queuecommand(struct Scsi_Host *host,
+ struct scsi_cmnd *sc_cmd);
+extern void qedf_fip_send(struct fcoe_ctlr *fip, struct sk_buff *skb);
+extern void qedf_update_src_mac(struct fc_lport *lport, u8 *addr);
+extern u8 *qedf_get_src_mac(struct fc_lport *lport);
+extern void qedf_fip_recv(struct qedf_ctx *qedf, struct sk_buff *skb);
+extern void qedf_fcoe_send_vlan_req(struct qedf_ctx *qedf);
+extern void qedf_scsi_completion(struct qedf_ctx *qedf, struct fcoe_cqe *cqe,
+ struct qedf_ioreq *io_req);
+extern void qedf_process_warning_compl(struct qedf_ctx *qedf,
+ struct fcoe_cqe *cqe, struct qedf_ioreq *io_req);
+extern void qedf_process_error_detect(struct qedf_ctx *qedf,
+ struct fcoe_cqe *cqe, struct qedf_ioreq *io_req);
+extern void qedf_flush_active_ios(struct qedf_rport *fcport, int lun);
+extern void qedf_release_cmd(struct kref *ref);
+extern int qedf_initiate_abts(struct qedf_ioreq *io_req,
+ bool return_scsi_cmd_on_abts);
+extern void qedf_process_abts_compl(struct qedf_ctx *qedf, struct fcoe_cqe *cqe,
+ struct qedf_ioreq *io_req);
+extern struct qedf_ioreq *qedf_alloc_cmd(struct qedf_rport *fcport,
+ u8 cmd_type);
+
+extern struct device_attribute *qedf_host_attrs[];
+extern void qedf_cmd_timer_set(struct qedf_ctx *qedf, struct qedf_ioreq *io_req,
+ unsigned int timer_msec);
+extern int qedf_init_mp_req(struct qedf_ioreq *io_req);
+extern void qedf_init_mp_task(struct qedf_ioreq *io_req,
+ struct fcoe_task_context *task_ctx);
+extern void qedf_add_to_sq(struct qedf_rport *fcport, u16 xid,
+ u32 ptu_invalidate, enum fcoe_task_type req_type, u32 offset);
+extern void qedf_ring_doorbell(struct qedf_rport *fcport);
+extern void qedf_process_els_compl(struct qedf_ctx *qedf, struct fcoe_cqe *cqe,
+ struct qedf_ioreq *els_req);
+extern int qedf_send_rrq(struct qedf_ioreq *aborted_io_req);
+extern int qedf_send_adisc(struct qedf_rport *fcport, struct fc_frame *fp);
+extern int qedf_initiate_cleanup(struct qedf_ioreq *io_req,
+ bool return_scsi_cmd_on_abts);
+extern void qedf_process_cleanup_compl(struct qedf_ctx *qedf,
+ struct fcoe_cqe *cqe, struct qedf_ioreq *io_req);
+extern int qedf_initiate_tmf(struct scsi_cmnd *sc_cmd, u8 tm_flags);
+extern void qedf_process_tmf_compl(struct qedf_ctx *qedf, struct fcoe_cqe *cqe,
+ struct qedf_ioreq *io_req);
+extern void qedf_process_cqe(struct qedf_ctx *qedf, struct fcoe_cqe *cqe);
+extern void qedf_scsi_done(struct qedf_ctx *qedf, struct qedf_ioreq *io_req,
+ int result);
+extern void qedf_set_vlan_id(struct qedf_ctx *qedf, int vlan_id);
+extern void qedf_create_sysfs_ctx_attr(struct qedf_ctx *qedf);
+extern void qedf_remove_sysfs_ctx_attr(struct qedf_ctx *qedf);
+extern void qedf_capture_grc_dump(struct qedf_ctx *qedf);
+extern void qedf_wait_for_upload(struct qedf_ctx *qedf);
+extern void qedf_process_unsol_compl(struct qedf_ctx *qedf, uint16_t que_idx,
+ struct fcoe_cqe *cqe);
+extern void qedf_restart_rport(struct qedf_rport *fcport);
+extern int qedf_send_rec(struct qedf_ioreq *orig_io_req);
+extern int qedf_post_io_req(struct qedf_rport *fcport,
+ struct qedf_ioreq *io_req);
+extern void qedf_process_seq_cleanup_compl(struct qedf_ctx *qedf,
+ struct fcoe_cqe *cqe, struct qedf_ioreq *io_req);
+extern int qedf_send_flogi(struct qedf_ctx *qedf);
+extern void qedf_fp_io_handler(struct work_struct *work);
+
+#define FCOE_WORD_TO_BYTE 4
+#define QEDF_MAX_TASK_NUM 0xFFFF
+
+struct fip_vlan {
+ struct ethhdr eth;
+ struct fip_header fip;
+ struct {
+ struct fip_mac_desc mac;
+ struct fip_wwn_desc wwnn;
+ } desc;
+};
+
+/* SQ/CQ Sizes */
+#define GBL_RSVD_TASKS 16
+#define NUM_TASKS_PER_CONNECTION 1024
+#define NUM_RW_TASKS_PER_CONNECTION 512
+#define FCOE_PARAMS_CQ_NUM_ENTRIES FCOE_PARAMS_NUM_TASKS
+
+#define FCOE_PARAMS_CMDQ_NUM_ENTRIES FCOE_PARAMS_NUM_TASKS
+#define SQ_NUM_ENTRIES NUM_TASKS_PER_CONNECTION
+
+#define QEDF_FCOE_PARAMS_GL_RQ_PI 0
+#define QEDF_FCOE_PARAMS_GL_CMD_PI 1
+
+#define QEDF_READ (1 << 1)
+#define QEDF_WRITE (1 << 0)
+#define MAX_FIBRE_LUNS 0xffffffff
+
+#define QEDF_MAX_NUM_CQS 8
+
+/*
+ * PCI function probe defines
+ */
+/* Probe/remove called during normal PCI probe */
+#define QEDF_MODE_NORMAL 0
+/* Probe/remove called from qed error recovery */
+#define QEDF_MODE_RECOVERY 1
+
+#define SUPPORTED_25000baseKR_Full (1<<27)
+#define SUPPORTED_50000baseKR2_Full (1<<28)
+#define SUPPORTED_100000baseKR4_Full (1<<29)
+#define SUPPORTED_100000baseCR4_Full (1<<30)
+
+#endif
--- /dev/null
+/*
+ * QLogic FCoE Offload Driver
+ * Copyright (c) 2016 Cavium Inc.
+ *
+ * This software is available under the terms of the GNU General Public License
+ * (GPL) Version 2, available from the file COPYING in the main directory of
+ * this source tree.
+ */
+#include "qedf.h"
+
+static ssize_t
+qedf_fcoe_mac_show(struct device *dev,
+ struct device_attribute *attr, char *buf)
+{
+ struct fc_lport *lport = shost_priv(class_to_shost(dev));
+ u32 port_id;
+ u8 lport_src_id[3];
+ u8 fcoe_mac[6];
+
+ port_id = fc_host_port_id(lport->host);
+ lport_src_id[2] = (port_id & 0x000000FF);
+ lport_src_id[1] = (port_id & 0x0000FF00) >> 8;
+ lport_src_id[0] = (port_id & 0x00FF0000) >> 16;
+ fc_fcoe_set_mac(fcoe_mac, lport_src_id);
+
+ return scnprintf(buf, PAGE_SIZE, "%pM\n", fcoe_mac);
+}
+
+static DEVICE_ATTR(fcoe_mac, S_IRUGO, qedf_fcoe_mac_show, NULL);
+
+struct device_attribute *qedf_host_attrs[] = {
+ &dev_attr_fcoe_mac,
+ NULL,
+};
+
+extern const struct qed_fcoe_ops *qed_ops;
+
+inline bool qedf_is_vport(struct qedf_ctx *qedf)
+{
+ return (!(qedf->lport->vport == NULL));
+}
+
+/* Get base qedf for physical port from vport */
+static struct qedf_ctx *qedf_get_base_qedf(struct qedf_ctx *qedf)
+{
+ struct fc_lport *lport;
+ struct fc_lport *base_lport;
+
+ if (!(qedf_is_vport(qedf)))
+ return NULL;
+
+ lport = qedf->lport;
+ base_lport = shost_priv(vport_to_shost(lport->vport));
+ return (struct qedf_ctx *)(lport_priv(base_lport));
+}
+
+void qedf_capture_grc_dump(struct qedf_ctx *qedf)
+{
+ struct qedf_ctx *base_qedf;
+
+ /* Make sure we use the base qedf to take the GRC dump */
+ if (qedf_is_vport(qedf))
+ base_qedf = qedf_get_base_qedf(qedf);
+ else
+ base_qedf = qedf;
+
+ if (test_bit(QEDF_GRCDUMP_CAPTURE, &base_qedf->flags)) {
+ QEDF_INFO(&(base_qedf->dbg_ctx), QEDF_LOG_INFO,
+ "GRC Dump already captured.\n");
+ return;
+ }
+
+
+ qedf_get_grc_dump(base_qedf->cdev, qed_ops->common,
+ &base_qedf->grcdump, &base_qedf->grcdump_size);
+ QEDF_ERR(&(base_qedf->dbg_ctx), "GRC Dump captured.\n");
+ set_bit(QEDF_GRCDUMP_CAPTURE, &base_qedf->flags);
+ qedf_uevent_emit(base_qedf->lport->host, QEDF_UEVENT_CODE_GRCDUMP,
+ NULL);
+}
+
+static ssize_t
+qedf_sysfs_read_grcdump(struct file *filep, struct kobject *kobj,
+ struct bin_attribute *ba, char *buf, loff_t off,
+ size_t count)
+{
+ ssize_t ret = 0;
+ struct fc_lport *lport = shost_priv(dev_to_shost(container_of(kobj,
+ struct device, kobj)));
+ struct qedf_ctx *qedf = lport_priv(lport);
+
+ if (test_bit(QEDF_GRCDUMP_CAPTURE, &qedf->flags)) {
+ ret = memory_read_from_buffer(buf, count, &off,
+ qedf->grcdump, qedf->grcdump_size);
+ } else {
+ QEDF_ERR(&(qedf->dbg_ctx), "GRC Dump not captured!\n");
+ }
+
+ return ret;
+}
+
+static ssize_t
+qedf_sysfs_write_grcdump(struct file *filep, struct kobject *kobj,
+ struct bin_attribute *ba, char *buf, loff_t off,
+ size_t count)
+{
+ struct fc_lport *lport = NULL;
+ struct qedf_ctx *qedf = NULL;
+ long reading;
+ int ret = 0;
+ char msg[40];
+
+ if (off != 0)
+ return ret;
+
+
+ lport = shost_priv(dev_to_shost(container_of(kobj,
+ struct device, kobj)));
+ qedf = lport_priv(lport);
+
+ buf[1] = 0;
+ ret = kstrtol(buf, 10, &reading);
+ if (ret) {
+ QEDF_ERR(&(qedf->dbg_ctx), "Invalid input, err(%d)\n", ret);
+ return ret;
+ }
+
+ memset(msg, 0, sizeof(msg));
+ switch (reading) {
+ case 0:
+ memset(qedf->grcdump, 0, qedf->grcdump_size);
+ clear_bit(QEDF_GRCDUMP_CAPTURE, &qedf->flags);
+ break;
+ case 1:
+ qedf_capture_grc_dump(qedf);
+ break;
+ }
+
+ return count;
+}
+
+static struct bin_attribute sysfs_grcdump_attr = {
+ .attr = {
+ .name = "grcdump",
+ .mode = S_IRUSR | S_IWUSR,
+ },
+ .size = 0,
+ .read = qedf_sysfs_read_grcdump,
+ .write = qedf_sysfs_write_grcdump,
+};
+
+static struct sysfs_bin_attrs bin_file_entries[] = {
+ {"grcdump", &sysfs_grcdump_attr},
+ {NULL},
+};
+
+void qedf_create_sysfs_ctx_attr(struct qedf_ctx *qedf)
+{
+ qedf_create_sysfs_attr(qedf->lport->host, bin_file_entries);
+}
+
+void qedf_remove_sysfs_ctx_attr(struct qedf_ctx *qedf)
+{
+ qedf_remove_sysfs_attr(qedf->lport->host, bin_file_entries);
+}
--- /dev/null
+/*
+ * QLogic FCoE Offload Driver
+ * Copyright (c) 2016 Cavium Inc.
+ *
+ * This software is available under the terms of the GNU General Public License
+ * (GPL) Version 2, available from the file COPYING in the main directory of
+ * this source tree.
+ */
+#include "qedf_dbg.h"
+#include <linux/vmalloc.h>
+
+void
+qedf_dbg_err(struct qedf_dbg_ctx *qedf, const char *func, u32 line,
+ const char *fmt, ...)
+{
+ va_list va;
+ struct va_format vaf;
+ char nfunc[32];
+
+ memset(nfunc, 0, sizeof(nfunc));
+ memcpy(nfunc, func, sizeof(nfunc) - 1);
+
+ va_start(va, fmt);
+
+ vaf.fmt = fmt;
+ vaf.va = &va;
+
+ if (likely(qedf) && likely(qedf->pdev))
+ pr_err("[%s]:[%s:%d]:%d: %pV", dev_name(&(qedf->pdev->dev)),
+ nfunc, line, qedf->host_no, &vaf);
+ else
+ pr_err("[0000:00:00.0]:[%s:%d]: %pV", nfunc, line, &vaf);
+
+ va_end(va);
+}
+
+void
+qedf_dbg_warn(struct qedf_dbg_ctx *qedf, const char *func, u32 line,
+ const char *fmt, ...)
+{
+ va_list va;
+ struct va_format vaf;
+ char nfunc[32];
+
+ memset(nfunc, 0, sizeof(nfunc));
+ memcpy(nfunc, func, sizeof(nfunc) - 1);
+
+ va_start(va, fmt);
+
+ vaf.fmt = fmt;
+ vaf.va = &va;
+
+ if (!(qedf_debug & QEDF_LOG_WARN))
+ goto ret;
+
+ if (likely(qedf) && likely(qedf->pdev))
+ pr_warn("[%s]:[%s:%d]:%d: %pV", dev_name(&(qedf->pdev->dev)),
+ nfunc, line, qedf->host_no, &vaf);
+ else
+ pr_warn("[0000:00:00.0]:[%s:%d]: %pV", nfunc, line, &vaf);
+
+ret:
+ va_end(va);
+}
+
+void
+qedf_dbg_notice(struct qedf_dbg_ctx *qedf, const char *func, u32 line,
+ const char *fmt, ...)
+{
+ va_list va;
+ struct va_format vaf;
+ char nfunc[32];
+
+ memset(nfunc, 0, sizeof(nfunc));
+ memcpy(nfunc, func, sizeof(nfunc) - 1);
+
+ va_start(va, fmt);
+
+ vaf.fmt = fmt;
+ vaf.va = &va;
+
+ if (!(qedf_debug & QEDF_LOG_NOTICE))
+ goto ret;
+
+ if (likely(qedf) && likely(qedf->pdev))
+ pr_notice("[%s]:[%s:%d]:%d: %pV",
+ dev_name(&(qedf->pdev->dev)), nfunc, line,
+ qedf->host_no, &vaf);
+ else
+ pr_notice("[0000:00:00.0]:[%s:%d]: %pV", nfunc, line, &vaf);
+
+ret:
+ va_end(va);
+}
+
+void
+qedf_dbg_info(struct qedf_dbg_ctx *qedf, const char *func, u32 line,
+ u32 level, const char *fmt, ...)
+{
+ va_list va;
+ struct va_format vaf;
+ char nfunc[32];
+
+ memset(nfunc, 0, sizeof(nfunc));
+ memcpy(nfunc, func, sizeof(nfunc) - 1);
+
+ va_start(va, fmt);
+
+ vaf.fmt = fmt;
+ vaf.va = &va;
+
+ if (!(qedf_debug & level))
+ goto ret;
+
+ if (likely(qedf) && likely(qedf->pdev))
+ pr_info("[%s]:[%s:%d]:%d: %pV", dev_name(&(qedf->pdev->dev)),
+ nfunc, line, qedf->host_no, &vaf);
+ else
+ pr_info("[0000:00:00.0]:[%s:%d]: %pV", nfunc, line, &vaf);
+
+ret:
+ va_end(va);
+}
+
+int
+qedf_alloc_grc_dump_buf(u8 **buf, uint32_t len)
+{
+ *buf = vmalloc(len);
+ if (!(*buf))
+ return -ENOMEM;
+
+ memset(*buf, 0, len);
+ return 0;
+}
+
+void
+qedf_free_grc_dump_buf(uint8_t **buf)
+{
+ vfree(*buf);
+ *buf = NULL;
+}
+
+int
+qedf_get_grc_dump(struct qed_dev *cdev, const struct qed_common_ops *common,
+ u8 **buf, uint32_t *grcsize)
+{
+ if (!*buf)
+ return -EINVAL;
+
+ return common->dbg_grc(cdev, *buf, grcsize);
+}
+
+void
+qedf_uevent_emit(struct Scsi_Host *shost, u32 code, char *msg)
+{
+ char event_string[40];
+ char *envp[] = {event_string, NULL};
+
+ memset(event_string, 0, sizeof(event_string));
+ switch (code) {
+ case QEDF_UEVENT_CODE_GRCDUMP:
+ if (msg)
+ strncpy(event_string, msg, strlen(msg));
+ else
+ sprintf(event_string, "GRCDUMP=%u", shost->host_no);
+ break;
+ default:
+ /* do nothing */
+ break;
+ }
+
+ kobject_uevent_env(&shost->shost_gendev.kobj, KOBJ_CHANGE, envp);
+}
+
+int
+qedf_create_sysfs_attr(struct Scsi_Host *shost, struct sysfs_bin_attrs *iter)
+{
+ int ret = 0;
+
+ for (; iter->name; iter++) {
+ ret = sysfs_create_bin_file(&shost->shost_gendev.kobj,
+ iter->attr);
+ if (ret)
+ pr_err("Unable to create sysfs %s attr, err(%d).\n",
+ iter->name, ret);
+ }
+ return ret;
+}
+
+void
+qedf_remove_sysfs_attr(struct Scsi_Host *shost, struct sysfs_bin_attrs *iter)
+{
+ for (; iter->name; iter++)
+ sysfs_remove_bin_file(&shost->shost_gendev.kobj, iter->attr);
+}
--- /dev/null
+/*
+ * QLogic FCoE Offload Driver
+ * Copyright (c) 2016 Cavium Inc.
+ *
+ * This software is available under the terms of the GNU General Public License
+ * (GPL) Version 2, available from the file COPYING in the main directory of
+ * this source tree.
+ */
+#ifndef _QEDF_DBG_H_
+#define _QEDF_DBG_H_
+
+#include <linux/types.h>
+#include <linux/kernel.h>
+#include <linux/compiler.h>
+#include <linux/string.h>
+#include <linux/version.h>
+#include <linux/pci.h>
+#include <linux/delay.h>
+#include <scsi/scsi_transport.h>
+#include <linux/fs.h>
+
+#include <linux/qed/common_hsi.h>
+#include <linux/qed/qed_if.h>
+
+extern uint qedf_debug;
+
+/* Debug print level definitions */
+#define QEDF_LOG_DEFAULT 0x1 /* Set default logging mask */
+#define QEDF_LOG_INFO 0x2 /*
+ * Informational logs,
+ * MAC address, WWPN, WWNN
+ */
+#define QEDF_LOG_DISC 0x4 /* Init, discovery, rport */
+#define QEDF_LOG_LL2 0x8 /* LL2, VLAN logs */
+#define QEDF_LOG_CONN 0x10 /* Connection setup, cleanup */
+#define QEDF_LOG_EVT 0x20 /* Events, link, mtu */
+#define QEDF_LOG_TIMER 0x40 /* Timer events */
+#define QEDF_LOG_MP_REQ 0x80 /* Middle Path (MP) logs */
+#define QEDF_LOG_SCSI_TM 0x100 /* SCSI Aborts, Task Mgmt */
+#define QEDF_LOG_UNSOL 0x200 /* unsolicited event logs */
+#define QEDF_LOG_IO 0x400 /* scsi cmd, completion */
+#define QEDF_LOG_MQ 0x800 /* Multi Queue logs */
+#define QEDF_LOG_BSG 0x1000 /* BSG logs */
+#define QEDF_LOG_DEBUGFS 0x2000 /* debugFS logs */
+#define QEDF_LOG_LPORT 0x4000 /* lport logs */
+#define QEDF_LOG_ELS 0x8000 /* ELS logs */
+#define QEDF_LOG_NPIV 0x10000 /* NPIV logs */
+#define QEDF_LOG_SESS 0x20000 /* Conection setup, cleanup */
+#define QEDF_LOG_TID 0x80000 /*
+ * FW TID context acquire
+ * free
+ */
+#define QEDF_TRACK_TID 0x100000 /*
+ * Track TID state. To be
+ * enabled only at module load
+ * and not run-time.
+ */
+#define QEDF_TRACK_CMD_LIST 0x300000 /*
+ * Track active cmd list nodes,
+ * done with reference to TID,
+ * hence TRACK_TID also enabled.
+ */
+#define QEDF_LOG_NOTICE 0x40000000 /* Notice logs */
+#define QEDF_LOG_WARN 0x80000000 /* Warning logs */
+
+/* Debug context structure */
+struct qedf_dbg_ctx {
+ unsigned int host_no;
+ struct pci_dev *pdev;
+#ifdef CONFIG_DEBUG_FS
+ struct dentry *bdf_dentry;
+#endif
+};
+
+#define QEDF_ERR(pdev, fmt, ...) \
+ qedf_dbg_err(pdev, __func__, __LINE__, fmt, ## __VA_ARGS__)
+#define QEDF_WARN(pdev, fmt, ...) \
+ qedf_dbg_warn(pdev, __func__, __LINE__, fmt, ## __VA_ARGS__)
+#define QEDF_NOTICE(pdev, fmt, ...) \
+ qedf_dbg_notice(pdev, __func__, __LINE__, fmt, ## __VA_ARGS__)
+#define QEDF_INFO(pdev, level, fmt, ...) \
+ qedf_dbg_info(pdev, __func__, __LINE__, level, fmt, \
+ ## __VA_ARGS__)
+
+extern void qedf_dbg_err(struct qedf_dbg_ctx *qedf, const char *func, u32 line,
+ const char *fmt, ...);
+extern void qedf_dbg_warn(struct qedf_dbg_ctx *qedf, const char *func, u32 line,
+ const char *, ...);
+extern void qedf_dbg_notice(struct qedf_dbg_ctx *qedf, const char *func,
+ u32 line, const char *, ...);
+extern void qedf_dbg_info(struct qedf_dbg_ctx *qedf, const char *func, u32 line,
+ u32 info, const char *fmt, ...);
+
+/* GRC Dump related defines */
+
+struct Scsi_Host;
+
+#define QEDF_UEVENT_CODE_GRCDUMP 0
+
+struct sysfs_bin_attrs {
+ char *name;
+ struct bin_attribute *attr;
+};
+
+extern int qedf_alloc_grc_dump_buf(uint8_t **buf, uint32_t len);
+extern void qedf_free_grc_dump_buf(uint8_t **buf);
+extern int qedf_get_grc_dump(struct qed_dev *cdev,
+ const struct qed_common_ops *common, uint8_t **buf,
+ uint32_t *grcsize);
+extern void qedf_uevent_emit(struct Scsi_Host *shost, u32 code, char *msg);
+extern int qedf_create_sysfs_attr(struct Scsi_Host *shost,
+ struct sysfs_bin_attrs *iter);
+extern void qedf_remove_sysfs_attr(struct Scsi_Host *shost,
+ struct sysfs_bin_attrs *iter);
+
+#ifdef CONFIG_DEBUG_FS
+/* DebugFS related code */
+struct qedf_list_of_funcs {
+ char *oper_str;
+ ssize_t (*oper_func)(struct qedf_dbg_ctx *qedf);
+};
+
+struct qedf_debugfs_ops {
+ char *name;
+ struct qedf_list_of_funcs *qedf_funcs;
+};
+
+#define qedf_dbg_fileops(drv, ops) \
+{ \
+ .owner = THIS_MODULE, \
+ .open = simple_open, \
+ .read = drv##_dbg_##ops##_cmd_read, \
+ .write = drv##_dbg_##ops##_cmd_write \
+}
+
+/* Used for debugfs sequential files */
+#define qedf_dbg_fileops_seq(drv, ops) \
+{ \
+ .owner = THIS_MODULE, \
+ .open = drv##_dbg_##ops##_open, \
+ .read = seq_read, \
+ .llseek = seq_lseek, \
+ .release = single_release, \
+}
+
+extern void qedf_dbg_host_init(struct qedf_dbg_ctx *qedf,
+ struct qedf_debugfs_ops *dops,
+ struct file_operations *fops);
+extern void qedf_dbg_host_exit(struct qedf_dbg_ctx *qedf);
+extern void qedf_dbg_init(char *drv_name);
+extern void qedf_dbg_exit(void);
+#endif /* CONFIG_DEBUG_FS */
+
+#endif /* _QEDF_DBG_H_ */
--- /dev/null
+/*
+ * QLogic FCoE Offload Driver
+ * Copyright (c) 2016 QLogic Corporation
+ *
+ * This software is available under the terms of the GNU General Public License
+ * (GPL) Version 2, available from the file COPYING in the main directory of
+ * this source tree.
+ */
+#ifdef CONFIG_DEBUG_FS
+
+#include <linux/uaccess.h>
+#include <linux/debugfs.h>
+#include <linux/module.h>
+
+#include "qedf.h"
+#include "qedf_dbg.h"
+
+static struct dentry *qedf_dbg_root;
+
+/**
+ * qedf_dbg_host_init - setup the debugfs file for the pf
+ * @pf: the pf that is starting up
+ **/
+void
+qedf_dbg_host_init(struct qedf_dbg_ctx *qedf,
+ struct qedf_debugfs_ops *dops,
+ struct file_operations *fops)
+{
+ char host_dirname[32];
+ struct dentry *file_dentry = NULL;
+
+ QEDF_INFO(qedf, QEDF_LOG_DEBUGFS, "Creating debugfs host node\n");
+ /* create pf dir */
+ sprintf(host_dirname, "host%u", qedf->host_no);
+ qedf->bdf_dentry = debugfs_create_dir(host_dirname, qedf_dbg_root);
+ if (!qedf->bdf_dentry)
+ return;
+
+ /* create debugfs files */
+ while (dops) {
+ if (!(dops->name))
+ break;
+
+ file_dentry = debugfs_create_file(dops->name, 0600,
+ qedf->bdf_dentry, qedf,
+ fops);
+ if (!file_dentry) {
+ QEDF_INFO(qedf, QEDF_LOG_DEBUGFS,
+ "Debugfs entry %s creation failed\n",
+ dops->name);
+ debugfs_remove_recursive(qedf->bdf_dentry);
+ return;
+ }
+ dops++;
+ fops++;
+ }
+}
+
+/**
+ * qedf_dbg_host_exit - clear out the pf's debugfs entries
+ * @pf: the pf that is stopping
+ **/
+void
+qedf_dbg_host_exit(struct qedf_dbg_ctx *qedf)
+{
+ QEDF_INFO(qedf, QEDF_LOG_DEBUGFS, "Destroying debugfs host "
+ "entry\n");
+ /* remove debugfs entries of this PF */
+ debugfs_remove_recursive(qedf->bdf_dentry);
+ qedf->bdf_dentry = NULL;
+}
+
+/**
+ * qedf_dbg_init - start up debugfs for the driver
+ **/
+void
+qedf_dbg_init(char *drv_name)
+{
+ QEDF_INFO(NULL, QEDF_LOG_DEBUGFS, "Creating debugfs root node\n");
+
+ /* create qed dir in root of debugfs. NULL means debugfs root */
+ qedf_dbg_root = debugfs_create_dir(drv_name, NULL);
+ if (!qedf_dbg_root)
+ QEDF_INFO(NULL, QEDF_LOG_DEBUGFS, "Init of debugfs "
+ "failed\n");
+}
+
+/**
+ * qedf_dbg_exit - clean out the driver's debugfs entries
+ **/
+void
+qedf_dbg_exit(void)
+{
+ QEDF_INFO(NULL, QEDF_LOG_DEBUGFS, "Destroying debugfs root "
+ "entry\n");
+
+ /* remove qed dir in root of debugfs */
+ debugfs_remove_recursive(qedf_dbg_root);
+ qedf_dbg_root = NULL;
+}
+
+struct qedf_debugfs_ops qedf_debugfs_ops[] = {
+ { "fp_int", NULL },
+ { "io_trace", NULL },
+ { "debug", NULL },
+ { "stop_io_on_error", NULL},
+ { "driver_stats", NULL},
+ { "clear_stats", NULL},
+ { "offload_stats", NULL},
+ /* This must be last */
+ { NULL, NULL }
+};
+
+DECLARE_PER_CPU(struct qedf_percpu_iothread_s, qedf_percpu_iothreads);
+
+static ssize_t
+qedf_dbg_fp_int_cmd_read(struct file *filp, char __user *buffer, size_t count,
+ loff_t *ppos)
+{
+ size_t cnt = 0;
+ int id;
+ struct qedf_fastpath *fp = NULL;
+ struct qedf_dbg_ctx *qedf_dbg =
+ (struct qedf_dbg_ctx *)filp->private_data;
+ struct qedf_ctx *qedf = container_of(qedf_dbg,
+ struct qedf_ctx, dbg_ctx);
+
+ QEDF_INFO(qedf_dbg, QEDF_LOG_DEBUGFS, "entered\n");
+
+ cnt = sprintf(buffer, "\nFastpath I/O completions\n\n");
+
+ for (id = 0; id < qedf->num_queues; id++) {
+ fp = &(qedf->fp_array[id]);
+ if (fp->sb_id == QEDF_SB_ID_NULL)
+ continue;
+ cnt += sprintf((buffer + cnt), "#%d: %lu\n", id,
+ fp->completions);
+ }
+
+ cnt = min_t(int, count, cnt - *ppos);
+ *ppos += cnt;
+ return cnt;
+}
+
+static ssize_t
+qedf_dbg_fp_int_cmd_write(struct file *filp, const char __user *buffer,
+ size_t count, loff_t *ppos)
+{
+ if (!count || *ppos)
+ return 0;
+
+ return count;
+}
+
+static ssize_t
+qedf_dbg_debug_cmd_read(struct file *filp, char __user *buffer, size_t count,
+ loff_t *ppos)
+{
+ int cnt;
+ struct qedf_dbg_ctx *qedf =
+ (struct qedf_dbg_ctx *)filp->private_data;
+
+ QEDF_INFO(qedf, QEDF_LOG_DEBUGFS, "entered\n");
+ cnt = sprintf(buffer, "debug mask = 0x%x\n", qedf_debug);
+
+ cnt = min_t(int, count, cnt - *ppos);
+ *ppos += cnt;
+ return cnt;
+}
+
+static ssize_t
+qedf_dbg_debug_cmd_write(struct file *filp, const char __user *buffer,
+ size_t count, loff_t *ppos)
+{
+ uint32_t val;
+ void *kern_buf;
+ int rval;
+ struct qedf_dbg_ctx *qedf =
+ (struct qedf_dbg_ctx *)filp->private_data;
+
+ if (!count || *ppos)
+ return 0;
+
+ kern_buf = memdup_user(buffer, count);
+ if (IS_ERR(kern_buf))
+ return PTR_ERR(kern_buf);
+
+ rval = kstrtouint(kern_buf, 10, &val);
+ kfree(kern_buf);
+ if (rval)
+ return rval;
+
+ if (val == 1)
+ qedf_debug = QEDF_DEFAULT_LOG_MASK;
+ else
+ qedf_debug = val;
+
+ QEDF_INFO(qedf, QEDF_LOG_DEBUGFS, "Setting debug=0x%x.\n", val);
+ return count;
+}
+
+static ssize_t
+qedf_dbg_stop_io_on_error_cmd_read(struct file *filp, char __user *buffer,
+ size_t count, loff_t *ppos)
+{
+ int cnt;
+ struct qedf_dbg_ctx *qedf_dbg =
+ (struct qedf_dbg_ctx *)filp->private_data;
+ struct qedf_ctx *qedf = container_of(qedf_dbg,
+ struct qedf_ctx, dbg_ctx);
+
+ QEDF_INFO(qedf_dbg, QEDF_LOG_DEBUGFS, "entered\n");
+ cnt = sprintf(buffer, "%s\n",
+ qedf->stop_io_on_error ? "true" : "false");
+
+ cnt = min_t(int, count, cnt - *ppos);
+ *ppos += cnt;
+ return cnt;
+}
+
+static ssize_t
+qedf_dbg_stop_io_on_error_cmd_write(struct file *filp,
+ const char __user *buffer, size_t count,
+ loff_t *ppos)
+{
+ void *kern_buf;
+ struct qedf_dbg_ctx *qedf_dbg =
+ (struct qedf_dbg_ctx *)filp->private_data;
+ struct qedf_ctx *qedf = container_of(qedf_dbg, struct qedf_ctx,
+ dbg_ctx);
+
+ QEDF_INFO(qedf_dbg, QEDF_LOG_DEBUGFS, "entered\n");
+
+ if (!count || *ppos)
+ return 0;
+
+ kern_buf = memdup_user(buffer, 6);
+ if (IS_ERR(kern_buf))
+ return PTR_ERR(kern_buf);
+
+ if (strncmp(kern_buf, "false", 5) == 0)
+ qedf->stop_io_on_error = false;
+ else if (strncmp(kern_buf, "true", 4) == 0)
+ qedf->stop_io_on_error = true;
+ else if (strncmp(kern_buf, "now", 3) == 0)
+ /* Trigger from user to stop all I/O on this host */
+ set_bit(QEDF_DBG_STOP_IO, &qedf->flags);
+
+ kfree(kern_buf);
+ return count;
+}
+
+static int
+qedf_io_trace_show(struct seq_file *s, void *unused)
+{
+ int i, idx = 0;
+ struct qedf_ctx *qedf = s->private;
+ struct qedf_dbg_ctx *qedf_dbg = &qedf->dbg_ctx;
+ struct qedf_io_log *io_log;
+ unsigned long flags;
+
+ if (!qedf_io_tracing) {
+ seq_puts(s, "I/O tracing not enabled.\n");
+ goto out;
+ }
+
+ QEDF_INFO(qedf_dbg, QEDF_LOG_DEBUGFS, "entered\n");
+
+ spin_lock_irqsave(&qedf->io_trace_lock, flags);
+ idx = qedf->io_trace_idx;
+ for (i = 0; i < QEDF_IO_TRACE_SIZE; i++) {
+ io_log = &qedf->io_trace_buf[idx];
+ seq_printf(s, "%d:", io_log->direction);
+ seq_printf(s, "0x%x:", io_log->task_id);
+ seq_printf(s, "0x%06x:", io_log->port_id);
+ seq_printf(s, "%d:", io_log->lun);
+ seq_printf(s, "0x%02x:", io_log->op);
+ seq_printf(s, "0x%02x%02x%02x%02x:", io_log->lba[0],
+ io_log->lba[1], io_log->lba[2], io_log->lba[3]);
+ seq_printf(s, "%d:", io_log->bufflen);
+ seq_printf(s, "%d:", io_log->sg_count);
+ seq_printf(s, "0x%08x:", io_log->result);
+ seq_printf(s, "%lu:", io_log->jiffies);
+ seq_printf(s, "%d:", io_log->refcount);
+ seq_printf(s, "%d:", io_log->req_cpu);
+ seq_printf(s, "%d:", io_log->int_cpu);
+ seq_printf(s, "%d:", io_log->rsp_cpu);
+ seq_printf(s, "%d\n", io_log->sge_type);
+
+ idx++;
+ if (idx == QEDF_IO_TRACE_SIZE)
+ idx = 0;
+ }
+ spin_unlock_irqrestore(&qedf->io_trace_lock, flags);
+
+out:
+ return 0;
+}
+
+static int
+qedf_dbg_io_trace_open(struct inode *inode, struct file *file)
+{
+ struct qedf_dbg_ctx *qedf_dbg = inode->i_private;
+ struct qedf_ctx *qedf = container_of(qedf_dbg,
+ struct qedf_ctx, dbg_ctx);
+
+ return single_open(file, qedf_io_trace_show, qedf);
+}
+
+static int
+qedf_driver_stats_show(struct seq_file *s, void *unused)
+{
+ struct qedf_ctx *qedf = s->private;
+ struct qedf_rport *fcport;
+ struct fc_rport_priv *rdata;
+
+ seq_printf(s, "cmg_mgr free io_reqs: %d\n",
+ atomic_read(&qedf->cmd_mgr->free_list_cnt));
+ seq_printf(s, "slow SGEs: %d\n", qedf->slow_sge_ios);
+ seq_printf(s, "single SGEs: %d\n", qedf->single_sge_ios);
+ seq_printf(s, "fast SGEs: %d\n\n", qedf->fast_sge_ios);
+
+ seq_puts(s, "Offloaded ports:\n\n");
+
+ rcu_read_lock();
+ list_for_each_entry_rcu(fcport, &qedf->fcports, peers) {
+ rdata = fcport->rdata;
+ if (rdata == NULL)
+ continue;
+ seq_printf(s, "%06x: free_sqes: %d, num_active_ios: %d\n",
+ rdata->ids.port_id, atomic_read(&fcport->free_sqes),
+ atomic_read(&fcport->num_active_ios));
+ }
+ rcu_read_unlock();
+
+ return 0;
+}
+
+static int
+qedf_dbg_driver_stats_open(struct inode *inode, struct file *file)
+{
+ struct qedf_dbg_ctx *qedf_dbg = inode->i_private;
+ struct qedf_ctx *qedf = container_of(qedf_dbg,
+ struct qedf_ctx, dbg_ctx);
+
+ return single_open(file, qedf_driver_stats_show, qedf);
+}
+
+static ssize_t
+qedf_dbg_clear_stats_cmd_read(struct file *filp, char __user *buffer,
+ size_t count, loff_t *ppos)
+{
+ int cnt = 0;
+
+ /* Essentially a read stub */
+ cnt = min_t(int, count, cnt - *ppos);
+ *ppos += cnt;
+ return cnt;
+}
+
+static ssize_t
+qedf_dbg_clear_stats_cmd_write(struct file *filp,
+ const char __user *buffer, size_t count,
+ loff_t *ppos)
+{
+ struct qedf_dbg_ctx *qedf_dbg =
+ (struct qedf_dbg_ctx *)filp->private_data;
+ struct qedf_ctx *qedf = container_of(qedf_dbg, struct qedf_ctx,
+ dbg_ctx);
+
+ QEDF_INFO(qedf_dbg, QEDF_LOG_DEBUGFS, "Clearing stat counters.\n");
+
+ if (!count || *ppos)
+ return 0;
+
+ /* Clear stat counters exposed by 'stats' node */
+ qedf->slow_sge_ios = 0;
+ qedf->single_sge_ios = 0;
+ qedf->fast_sge_ios = 0;
+
+ return count;
+}
+
+static int
+qedf_offload_stats_show(struct seq_file *s, void *unused)
+{
+ struct qedf_ctx *qedf = s->private;
+ struct qed_fcoe_stats *fw_fcoe_stats;
+
+ fw_fcoe_stats = kmalloc(sizeof(struct qed_fcoe_stats), GFP_KERNEL);
+ if (!fw_fcoe_stats) {
+ QEDF_ERR(&(qedf->dbg_ctx), "Could not allocate memory for "
+ "fw_fcoe_stats.\n");
+ goto out;
+ }
+
+ /* Query firmware for offload stats */
+ qed_ops->get_stats(qedf->cdev, fw_fcoe_stats);
+
+ seq_printf(s, "fcoe_rx_byte_cnt=%llu\n"
+ "fcoe_rx_data_pkt_cnt=%llu\n"
+ "fcoe_rx_xfer_pkt_cnt=%llu\n"
+ "fcoe_rx_other_pkt_cnt=%llu\n"
+ "fcoe_silent_drop_pkt_cmdq_full_cnt=%u\n"
+ "fcoe_silent_drop_pkt_crc_error_cnt=%u\n"
+ "fcoe_silent_drop_pkt_task_invalid_cnt=%u\n"
+ "fcoe_silent_drop_total_pkt_cnt=%u\n"
+ "fcoe_silent_drop_pkt_rq_full_cnt=%u\n"
+ "fcoe_tx_byte_cnt=%llu\n"
+ "fcoe_tx_data_pkt_cnt=%llu\n"
+ "fcoe_tx_xfer_pkt_cnt=%llu\n"
+ "fcoe_tx_other_pkt_cnt=%llu\n",
+ fw_fcoe_stats->fcoe_rx_byte_cnt,
+ fw_fcoe_stats->fcoe_rx_data_pkt_cnt,
+ fw_fcoe_stats->fcoe_rx_xfer_pkt_cnt,
+ fw_fcoe_stats->fcoe_rx_other_pkt_cnt,
+ fw_fcoe_stats->fcoe_silent_drop_pkt_cmdq_full_cnt,
+ fw_fcoe_stats->fcoe_silent_drop_pkt_crc_error_cnt,
+ fw_fcoe_stats->fcoe_silent_drop_pkt_task_invalid_cnt,
+ fw_fcoe_stats->fcoe_silent_drop_total_pkt_cnt,
+ fw_fcoe_stats->fcoe_silent_drop_pkt_rq_full_cnt,
+ fw_fcoe_stats->fcoe_tx_byte_cnt,
+ fw_fcoe_stats->fcoe_tx_data_pkt_cnt,
+ fw_fcoe_stats->fcoe_tx_xfer_pkt_cnt,
+ fw_fcoe_stats->fcoe_tx_other_pkt_cnt);
+
+ kfree(fw_fcoe_stats);
+out:
+ return 0;
+}
+
+static int
+qedf_dbg_offload_stats_open(struct inode *inode, struct file *file)
+{
+ struct qedf_dbg_ctx *qedf_dbg = inode->i_private;
+ struct qedf_ctx *qedf = container_of(qedf_dbg,
+ struct qedf_ctx, dbg_ctx);
+
+ return single_open(file, qedf_offload_stats_show, qedf);
+}
+
+
+const struct file_operations qedf_dbg_fops[] = {
+ qedf_dbg_fileops(qedf, fp_int),
+ qedf_dbg_fileops_seq(qedf, io_trace),
+ qedf_dbg_fileops(qedf, debug),
+ qedf_dbg_fileops(qedf, stop_io_on_error),
+ qedf_dbg_fileops_seq(qedf, driver_stats),
+ qedf_dbg_fileops(qedf, clear_stats),
+ qedf_dbg_fileops_seq(qedf, offload_stats),
+ /* This must be last */
+ { NULL, NULL },
+};
+
+#else /* CONFIG_DEBUG_FS */
+void qedf_dbg_host_init(struct qedf_dbg_ctx *);
+void qedf_dbg_host_exit(struct qedf_dbg_ctx *);
+void qedf_dbg_init(char *);
+void qedf_dbg_exit(void);
+#endif /* CONFIG_DEBUG_FS */
--- /dev/null
+/*
+ * QLogic FCoE Offload Driver
+ * Copyright (c) 2016 Cavium Inc.
+ *
+ * This software is available under the terms of the GNU General Public License
+ * (GPL) Version 2, available from the file COPYING in the main directory of
+ * this source tree.
+ */
+#include "qedf.h"
+
+/* It's assumed that the lock is held when calling this function. */
+static int qedf_initiate_els(struct qedf_rport *fcport, unsigned int op,
+ void *data, uint32_t data_len,
+ void (*cb_func)(struct qedf_els_cb_arg *cb_arg),
+ struct qedf_els_cb_arg *cb_arg, uint32_t timer_msec)
+{
+ struct qedf_ctx *qedf = fcport->qedf;
+ struct fc_lport *lport = qedf->lport;
+ struct qedf_ioreq *els_req;
+ struct qedf_mp_req *mp_req;
+ struct fc_frame_header *fc_hdr;
+ struct fcoe_task_context *task;
+ int rc = 0;
+ uint32_t did, sid;
+ uint16_t xid;
+ uint32_t start_time = jiffies / HZ;
+ uint32_t current_time;
+
+ QEDF_INFO(&(qedf->dbg_ctx), QEDF_LOG_ELS, "Sending ELS\n");
+
+ rc = fc_remote_port_chkready(fcport->rport);
+ if (rc) {
+ QEDF_ERR(&(qedf->dbg_ctx), "els 0x%x: rport not ready\n", op);
+ rc = -EAGAIN;
+ goto els_err;
+ }
+ if (lport->state != LPORT_ST_READY || !(lport->link_up)) {
+ QEDF_ERR(&(qedf->dbg_ctx), "els 0x%x: link is not ready\n",
+ op);
+ rc = -EAGAIN;
+ goto els_err;
+ }
+
+ if (!(test_bit(QEDF_RPORT_SESSION_READY, &fcport->flags))) {
+ QEDF_ERR(&(qedf->dbg_ctx), "els 0x%x: fcport not ready\n", op);
+ rc = -EINVAL;
+ goto els_err;
+ }
+
+retry_els:
+ els_req = qedf_alloc_cmd(fcport, QEDF_ELS);
+ if (!els_req) {
+ current_time = jiffies / HZ;
+ if ((current_time - start_time) > 10) {
+ QEDF_INFO(&(qedf->dbg_ctx), QEDF_LOG_ELS,
+ "els: Failed els 0x%x\n", op);
+ rc = -ENOMEM;
+ goto els_err;
+ }
+ mdelay(20 * USEC_PER_MSEC);
+ goto retry_els;
+ }
+
+ QEDF_INFO(&(qedf->dbg_ctx), QEDF_LOG_ELS, "initiate_els els_req = "
+ "0x%p cb_arg = %p xid = %x\n", els_req, cb_arg,
+ els_req->xid);
+ els_req->sc_cmd = NULL;
+ els_req->cmd_type = QEDF_ELS;
+ els_req->fcport = fcport;
+ els_req->cb_func = cb_func;
+ cb_arg->io_req = els_req;
+ cb_arg->op = op;
+ els_req->cb_arg = cb_arg;
+ els_req->data_xfer_len = data_len;
+
+ /* Record which cpu this request is associated with */
+ els_req->cpu = smp_processor_id();
+
+ mp_req = (struct qedf_mp_req *)&(els_req->mp_req);
+ rc = qedf_init_mp_req(els_req);
+ if (rc) {
+ QEDF_ERR(&(qedf->dbg_ctx), "ELS MP request init failed\n");
+ kref_put(&els_req->refcount, qedf_release_cmd);
+ goto els_err;
+ } else {
+ rc = 0;
+ }
+
+ /* Fill ELS Payload */
+ if ((op >= ELS_LS_RJT) && (op <= ELS_AUTH_ELS)) {
+ memcpy(mp_req->req_buf, data, data_len);
+ } else {
+ QEDF_ERR(&(qedf->dbg_ctx), "Invalid ELS op 0x%x\n", op);
+ els_req->cb_func = NULL;
+ els_req->cb_arg = NULL;
+ kref_put(&els_req->refcount, qedf_release_cmd);
+ rc = -EINVAL;
+ }
+
+ if (rc)
+ goto els_err;
+
+ /* Fill FC header */
+ fc_hdr = &(mp_req->req_fc_hdr);
+
+ did = fcport->rdata->ids.port_id;
+ sid = fcport->sid;
+
+ __fc_fill_fc_hdr(fc_hdr, FC_RCTL_ELS_REQ, sid, did,
+ FC_TYPE_ELS, FC_FC_FIRST_SEQ | FC_FC_END_SEQ |
+ FC_FC_SEQ_INIT, 0);
+
+ /* Obtain exchange id */
+ xid = els_req->xid;
+
+ /* Initialize task context for this IO request */
+ task = qedf_get_task_mem(&qedf->tasks, xid);
+ qedf_init_mp_task(els_req, task);
+
+ /* Put timer on original I/O request */
+ if (timer_msec)
+ qedf_cmd_timer_set(qedf, els_req, timer_msec);
+
+ qedf_add_to_sq(fcport, xid, 0, FCOE_TASK_TYPE_MIDPATH, 0);
+
+ /* Ring doorbell */
+ QEDF_INFO(&(qedf->dbg_ctx), QEDF_LOG_ELS, "Ringing doorbell for ELS "
+ "req\n");
+ qedf_ring_doorbell(fcport);
+els_err:
+ return rc;
+}
+
+void qedf_process_els_compl(struct qedf_ctx *qedf, struct fcoe_cqe *cqe,
+ struct qedf_ioreq *els_req)
+{
+ struct fcoe_task_context *task_ctx;
+ struct scsi_cmnd *sc_cmd;
+ uint16_t xid;
+ struct fcoe_cqe_midpath_info *mp_info;
+
+ QEDF_INFO(&(qedf->dbg_ctx), QEDF_LOG_ELS, "Entered with xid = 0x%x"
+ " cmd_type = %d.\n", els_req->xid, els_req->cmd_type);
+
+ /* Kill the ELS timer */
+ cancel_delayed_work(&els_req->timeout_work);
+
+ xid = els_req->xid;
+ task_ctx = qedf_get_task_mem(&qedf->tasks, xid);
+ sc_cmd = els_req->sc_cmd;
+
+ /* Get ELS response length from CQE */
+ mp_info = &cqe->cqe_info.midpath_info;
+ els_req->mp_req.resp_len = mp_info->data_placement_size;
+
+ /* Parse ELS response */
+ if ((els_req->cb_func) && (els_req->cb_arg)) {
+ els_req->cb_func(els_req->cb_arg);
+ els_req->cb_arg = NULL;
+ }
+
+ kref_put(&els_req->refcount, qedf_release_cmd);
+}
+
+static void qedf_rrq_compl(struct qedf_els_cb_arg *cb_arg)
+{
+ struct qedf_ioreq *orig_io_req;
+ struct qedf_ioreq *rrq_req;
+ struct qedf_ctx *qedf;
+ int refcount;
+
+ rrq_req = cb_arg->io_req;
+ qedf = rrq_req->fcport->qedf;
+
+ QEDF_INFO(&(qedf->dbg_ctx), QEDF_LOG_ELS, "Entered.\n");
+
+ orig_io_req = cb_arg->aborted_io_req;
+
+ if (!orig_io_req)
+ goto out_free;
+
+ if (rrq_req->event != QEDF_IOREQ_EV_ELS_TMO &&
+ rrq_req->event != QEDF_IOREQ_EV_ELS_ERR_DETECT)
+ cancel_delayed_work_sync(&orig_io_req->timeout_work);
+
+ refcount = kref_read(&orig_io_req->refcount);
+ QEDF_INFO(&(qedf->dbg_ctx), QEDF_LOG_ELS, "rrq_compl: orig io = %p,"
+ " orig xid = 0x%x, rrq_xid = 0x%x, refcount=%d\n",
+ orig_io_req, orig_io_req->xid, rrq_req->xid, refcount);
+
+ /* This should return the aborted io_req to the command pool */
+ if (orig_io_req)
+ kref_put(&orig_io_req->refcount, qedf_release_cmd);
+
+out_free:
+ kfree(cb_arg);
+}
+
+/* Assumes kref is already held by caller */
+int qedf_send_rrq(struct qedf_ioreq *aborted_io_req)
+{
+
+ struct fc_els_rrq rrq;
+ struct qedf_rport *fcport;
+ struct fc_lport *lport;
+ struct qedf_els_cb_arg *cb_arg = NULL;
+ struct qedf_ctx *qedf;
+ uint32_t sid;
+ uint32_t r_a_tov;
+ int rc;
+
+ if (!aborted_io_req) {
+ QEDF_ERR(NULL, "abort_io_req is NULL.\n");
+ return -EINVAL;
+ }
+
+ fcport = aborted_io_req->fcport;
+
+ /* Check that fcport is still offloaded */
+ if (!(test_bit(QEDF_RPORT_SESSION_READY, &fcport->flags))) {
+ QEDF_ERR(NULL, "fcport is no longer offloaded.\n");
+ return -EINVAL;
+ }
+
+ if (!fcport->qedf) {
+ QEDF_ERR(NULL, "fcport->qedf is NULL.\n");
+ return -EINVAL;
+ }
+
+ qedf = fcport->qedf;
+ lport = qedf->lport;
+ sid = fcport->sid;
+ r_a_tov = lport->r_a_tov;
+
+ QEDF_INFO(&(qedf->dbg_ctx), QEDF_LOG_ELS, "Sending RRQ orig "
+ "io = %p, orig_xid = 0x%x\n", aborted_io_req,
+ aborted_io_req->xid);
+ memset(&rrq, 0, sizeof(rrq));
+
+ cb_arg = kzalloc(sizeof(struct qedf_els_cb_arg), GFP_NOIO);
+ if (!cb_arg) {
+ QEDF_ERR(&(qedf->dbg_ctx), "Unable to allocate cb_arg for "
+ "RRQ\n");
+ rc = -ENOMEM;
+ goto rrq_err;
+ }
+
+ cb_arg->aborted_io_req = aborted_io_req;
+
+ rrq.rrq_cmd = ELS_RRQ;
+ hton24(rrq.rrq_s_id, sid);
+ rrq.rrq_ox_id = htons(aborted_io_req->xid);
+ rrq.rrq_rx_id =
+ htons(aborted_io_req->task->tstorm_st_context.read_write.rx_id);
+
+ rc = qedf_initiate_els(fcport, ELS_RRQ, &rrq, sizeof(rrq),
+ qedf_rrq_compl, cb_arg, r_a_tov);
+
+rrq_err:
+ if (rc) {
+ QEDF_ERR(&(qedf->dbg_ctx), "RRQ failed - release orig io "
+ "req 0x%x\n", aborted_io_req->xid);
+ kfree(cb_arg);
+ kref_put(&aborted_io_req->refcount, qedf_release_cmd);
+ }
+ return rc;
+}
+
+static void qedf_process_l2_frame_compl(struct qedf_rport *fcport,
+ struct fc_frame *fp,
+ u16 l2_oxid)
+{
+ struct fc_lport *lport = fcport->qedf->lport;
+ struct fc_frame_header *fh;
+ u32 crc;
+
+ fh = (struct fc_frame_header *)fc_frame_header_get(fp);
+
+ /* Set the OXID we return to what libfc used */
+ if (l2_oxid != FC_XID_UNKNOWN)
+ fh->fh_ox_id = htons(l2_oxid);
+
+ /* Setup header fields */
+ fh->fh_r_ctl = FC_RCTL_ELS_REP;
+ fh->fh_type = FC_TYPE_ELS;
+ /* Last sequence, end sequence */
+ fh->fh_f_ctl[0] = 0x98;
+ hton24(fh->fh_d_id, lport->port_id);
+ hton24(fh->fh_s_id, fcport->rdata->ids.port_id);
+ fh->fh_rx_id = 0xffff;
+
+ /* Set frame attributes */
+ crc = fcoe_fc_crc(fp);
+ fc_frame_init(fp);
+ fr_dev(fp) = lport;
+ fr_sof(fp) = FC_SOF_I3;
+ fr_eof(fp) = FC_EOF_T;
+ fr_crc(fp) = cpu_to_le32(~crc);
+
+ /* Send completed request to libfc */
+ fc_exch_recv(lport, fp);
+}
+
+/*
+ * In instances where an ELS command times out we may need to restart the
+ * rport by logging out and then logging back in.
+ */
+void qedf_restart_rport(struct qedf_rport *fcport)
+{
+ struct fc_lport *lport;
+ struct fc_rport_priv *rdata;
+ u32 port_id;
+
+ if (!fcport)
+ return;
+
+ rdata = fcport->rdata;
+ if (rdata) {
+ lport = fcport->qedf->lport;
+ port_id = rdata->ids.port_id;
+ QEDF_ERR(&(fcport->qedf->dbg_ctx),
+ "LOGO port_id=%x.\n", port_id);
+ fc_rport_logoff(rdata);
+ /* Recreate the rport and log back in */
+ rdata = fc_rport_create(lport, port_id);
+ if (rdata)
+ fc_rport_login(rdata);
+ }
+}
+
+static void qedf_l2_els_compl(struct qedf_els_cb_arg *cb_arg)
+{
+ struct qedf_ioreq *els_req;
+ struct qedf_rport *fcport;
+ struct qedf_mp_req *mp_req;
+ struct fc_frame *fp;
+ struct fc_frame_header *fh, *mp_fc_hdr;
+ void *resp_buf, *fc_payload;
+ u32 resp_len;
+ u16 l2_oxid;
+
+ l2_oxid = cb_arg->l2_oxid;
+ els_req = cb_arg->io_req;
+
+ if (!els_req) {
+ QEDF_ERR(NULL, "els_req is NULL.\n");
+ goto free_arg;
+ }
+
+ /*
+ * If we are flushing the command just free the cb_arg as none of the
+ * response data will be valid.
+ */
+ if (els_req->event == QEDF_IOREQ_EV_ELS_FLUSH)
+ goto free_arg;
+
+ fcport = els_req->fcport;
+ mp_req = &(els_req->mp_req);
+ mp_fc_hdr = &(mp_req->resp_fc_hdr);
+ resp_len = mp_req->resp_len;
+ resp_buf = mp_req->resp_buf;
+
+ /*
+ * If a middle path ELS command times out, don't try to return
+ * the command but rather do any internal cleanup and then libfc
+ * timeout the command and clean up its internal resources.
+ */
+ if (els_req->event == QEDF_IOREQ_EV_ELS_TMO) {
+ /*
+ * If ADISC times out, libfc will timeout the exchange and then
+ * try to send a PLOGI which will timeout since the session is
+ * still offloaded. Force libfc to logout the session which
+ * will offload the connection and allow the PLOGI response to
+ * flow over the LL2 path.
+ */
+ if (cb_arg->op == ELS_ADISC)
+ qedf_restart_rport(fcport);
+ return;
+ }
+
+ if (sizeof(struct fc_frame_header) + resp_len > QEDF_PAGE_SIZE) {
+ QEDF_ERR(&(fcport->qedf->dbg_ctx), "resp_len is "
+ "beyond page size.\n");
+ goto free_arg;
+ }
+
+ fp = fc_frame_alloc(fcport->qedf->lport, resp_len);
+ if (!fp) {
+ QEDF_ERR(&(fcport->qedf->dbg_ctx),
+ "fc_frame_alloc failure.\n");
+ return;
+ }
+
+ /* Copy frame header from firmware into fp */
+ fh = (struct fc_frame_header *)fc_frame_header_get(fp);
+ memcpy(fh, mp_fc_hdr, sizeof(struct fc_frame_header));
+
+ /* Copy payload from firmware into fp */
+ fc_payload = fc_frame_payload_get(fp, resp_len);
+ memcpy(fc_payload, resp_buf, resp_len);
+
+ QEDF_INFO(&(fcport->qedf->dbg_ctx), QEDF_LOG_ELS,
+ "Completing OX_ID 0x%x back to libfc.\n", l2_oxid);
+ qedf_process_l2_frame_compl(fcport, fp, l2_oxid);
+
+free_arg:
+ kfree(cb_arg);
+}
+
+int qedf_send_adisc(struct qedf_rport *fcport, struct fc_frame *fp)
+{
+ struct fc_els_adisc *adisc;
+ struct fc_frame_header *fh;
+ struct fc_lport *lport = fcport->qedf->lport;
+ struct qedf_els_cb_arg *cb_arg = NULL;
+ struct qedf_ctx *qedf;
+ uint32_t r_a_tov = lport->r_a_tov;
+ int rc;
+
+ qedf = fcport->qedf;
+ fh = fc_frame_header_get(fp);
+
+ cb_arg = kzalloc(sizeof(struct qedf_els_cb_arg), GFP_NOIO);
+ if (!cb_arg) {
+ QEDF_ERR(&(qedf->dbg_ctx), "Unable to allocate cb_arg for "
+ "ADISC\n");
+ rc = -ENOMEM;
+ goto adisc_err;
+ }
+ cb_arg->l2_oxid = ntohs(fh->fh_ox_id);
+
+ QEDF_INFO(&(qedf->dbg_ctx), QEDF_LOG_ELS,
+ "Sending ADISC ox_id=0x%x.\n", cb_arg->l2_oxid);
+
+ adisc = fc_frame_payload_get(fp, sizeof(*adisc));
+
+ rc = qedf_initiate_els(fcport, ELS_ADISC, adisc, sizeof(*adisc),
+ qedf_l2_els_compl, cb_arg, r_a_tov);
+
+adisc_err:
+ if (rc) {
+ QEDF_ERR(&(qedf->dbg_ctx), "ADISC failed.\n");
+ kfree(cb_arg);
+ }
+ return rc;
+}
+
+static void qedf_srr_compl(struct qedf_els_cb_arg *cb_arg)
+{
+ struct qedf_ioreq *orig_io_req;
+ struct qedf_ioreq *srr_req;
+ struct qedf_mp_req *mp_req;
+ struct fc_frame_header *mp_fc_hdr, *fh;
+ struct fc_frame *fp;
+ void *resp_buf, *fc_payload;
+ u32 resp_len;
+ struct fc_lport *lport;
+ struct qedf_ctx *qedf;
+ int refcount;
+ u8 opcode;
+
+ srr_req = cb_arg->io_req;
+ qedf = srr_req->fcport->qedf;
+ lport = qedf->lport;
+
+ orig_io_req = cb_arg->aborted_io_req;
+
+ if (!orig_io_req)
+ goto out_free;
+
+ clear_bit(QEDF_CMD_SRR_SENT, &orig_io_req->flags);
+
+ if (srr_req->event != QEDF_IOREQ_EV_ELS_TMO &&
+ srr_req->event != QEDF_IOREQ_EV_ELS_ERR_DETECT)
+ cancel_delayed_work_sync(&orig_io_req->timeout_work);
+
+ refcount = kref_read(&orig_io_req->refcount);
+ QEDF_INFO(&(qedf->dbg_ctx), QEDF_LOG_ELS, "Entered: orig_io=%p,"
+ " orig_io_xid=0x%x, rec_xid=0x%x, refcount=%d\n",
+ orig_io_req, orig_io_req->xid, srr_req->xid, refcount);
+
+ /* If a SRR times out, simply free resources */
+ if (srr_req->event == QEDF_IOREQ_EV_ELS_TMO)
+ goto out_free;
+
+ /* Normalize response data into struct fc_frame */
+ mp_req = &(srr_req->mp_req);
+ mp_fc_hdr = &(mp_req->resp_fc_hdr);
+ resp_len = mp_req->resp_len;
+ resp_buf = mp_req->resp_buf;
+
+ fp = fc_frame_alloc(lport, resp_len);
+ if (!fp) {
+ QEDF_ERR(&(qedf->dbg_ctx),
+ "fc_frame_alloc failure.\n");
+ goto out_free;
+ }
+
+ /* Copy frame header from firmware into fp */
+ fh = (struct fc_frame_header *)fc_frame_header_get(fp);
+ memcpy(fh, mp_fc_hdr, sizeof(struct fc_frame_header));
+
+ /* Copy payload from firmware into fp */
+ fc_payload = fc_frame_payload_get(fp, resp_len);
+ memcpy(fc_payload, resp_buf, resp_len);
+
+ opcode = fc_frame_payload_op(fp);
+ switch (opcode) {
+ case ELS_LS_ACC:
+ QEDF_INFO(&(qedf->dbg_ctx), QEDF_LOG_ELS,
+ "SRR success.\n");
+ break;
+ case ELS_LS_RJT:
+ QEDF_INFO(&qedf->dbg_ctx, QEDF_LOG_ELS,
+ "SRR rejected.\n");
+ qedf_initiate_abts(orig_io_req, true);
+ break;
+ }
+
+ fc_frame_free(fp);
+out_free:
+ /* Put reference for original command since SRR completed */
+ kref_put(&orig_io_req->refcount, qedf_release_cmd);
+ kfree(cb_arg);
+}
+
+static int qedf_send_srr(struct qedf_ioreq *orig_io_req, u32 offset, u8 r_ctl)
+{
+ struct fcp_srr srr;
+ struct qedf_ctx *qedf;
+ struct qedf_rport *fcport;
+ struct fc_lport *lport;
+ struct qedf_els_cb_arg *cb_arg = NULL;
+ u32 sid, r_a_tov;
+ int rc;
+
+ if (!orig_io_req) {
+ QEDF_ERR(NULL, "orig_io_req is NULL.\n");
+ return -EINVAL;
+ }
+
+ fcport = orig_io_req->fcport;
+
+ /* Check that fcport is still offloaded */
+ if (!(test_bit(QEDF_RPORT_SESSION_READY, &fcport->flags))) {
+ QEDF_ERR(NULL, "fcport is no longer offloaded.\n");
+ return -EINVAL;
+ }
+
+ if (!fcport->qedf) {
+ QEDF_ERR(NULL, "fcport->qedf is NULL.\n");
+ return -EINVAL;
+ }
+
+ /* Take reference until SRR command completion */
+ kref_get(&orig_io_req->refcount);
+
+ qedf = fcport->qedf;
+ lport = qedf->lport;
+ sid = fcport->sid;
+ r_a_tov = lport->r_a_tov;
+
+ QEDF_INFO(&(qedf->dbg_ctx), QEDF_LOG_ELS, "Sending SRR orig_io=%p, "
+ "orig_xid=0x%x\n", orig_io_req, orig_io_req->xid);
+ memset(&srr, 0, sizeof(srr));
+
+ cb_arg = kzalloc(sizeof(struct qedf_els_cb_arg), GFP_NOIO);
+ if (!cb_arg) {
+ QEDF_ERR(&(qedf->dbg_ctx), "Unable to allocate cb_arg for "
+ "SRR\n");
+ rc = -ENOMEM;
+ goto srr_err;
+ }
+
+ cb_arg->aborted_io_req = orig_io_req;
+
+ srr.srr_op = ELS_SRR;
+ srr.srr_ox_id = htons(orig_io_req->xid);
+ srr.srr_rx_id = htons(orig_io_req->rx_id);
+ srr.srr_rel_off = htonl(offset);
+ srr.srr_r_ctl = r_ctl;
+
+ rc = qedf_initiate_els(fcport, ELS_SRR, &srr, sizeof(srr),
+ qedf_srr_compl, cb_arg, r_a_tov);
+
+srr_err:
+ if (rc) {
+ QEDF_ERR(&(qedf->dbg_ctx), "SRR failed - release orig_io_req"
+ "=0x%x\n", orig_io_req->xid);
+ kfree(cb_arg);
+ /* If we fail to queue SRR, send ABTS to orig_io */
+ qedf_initiate_abts(orig_io_req, true);
+ kref_put(&orig_io_req->refcount, qedf_release_cmd);
+ } else
+ /* Tell other threads that SRR is in progress */
+ set_bit(QEDF_CMD_SRR_SENT, &orig_io_req->flags);
+
+ return rc;
+}
+
+static void qedf_initiate_seq_cleanup(struct qedf_ioreq *orig_io_req,
+ u32 offset, u8 r_ctl)
+{
+ struct qedf_rport *fcport;
+ unsigned long flags;
+ struct qedf_els_cb_arg *cb_arg;
+
+ fcport = orig_io_req->fcport;
+
+ QEDF_INFO(&(fcport->qedf->dbg_ctx), QEDF_LOG_ELS,
+ "Doing sequence cleanup for xid=0x%x offset=%u.\n",
+ orig_io_req->xid, offset);
+
+ cb_arg = kzalloc(sizeof(struct qedf_els_cb_arg), GFP_NOIO);
+ if (!cb_arg) {
+ QEDF_ERR(&(fcport->qedf->dbg_ctx), "Unable to allocate cb_arg "
+ "for sequence cleanup\n");
+ return;
+ }
+
+ /* Get reference for cleanup request */
+ kref_get(&orig_io_req->refcount);
+
+ orig_io_req->cmd_type = QEDF_SEQ_CLEANUP;
+ cb_arg->offset = offset;
+ cb_arg->r_ctl = r_ctl;
+ orig_io_req->cb_arg = cb_arg;
+
+ qedf_cmd_timer_set(fcport->qedf, orig_io_req,
+ QEDF_CLEANUP_TIMEOUT * HZ);
+
+ spin_lock_irqsave(&fcport->rport_lock, flags);
+
+ qedf_add_to_sq(fcport, orig_io_req->xid, 0,
+ FCOE_TASK_TYPE_SEQUENCE_CLEANUP, offset);
+ qedf_ring_doorbell(fcport);
+
+ spin_unlock_irqrestore(&fcport->rport_lock, flags);
+}
+
+void qedf_process_seq_cleanup_compl(struct qedf_ctx *qedf,
+ struct fcoe_cqe *cqe, struct qedf_ioreq *io_req)
+{
+ int rc;
+ struct qedf_els_cb_arg *cb_arg;
+
+ cb_arg = io_req->cb_arg;
+
+ /* If we timed out just free resources */
+ if (io_req->event == QEDF_IOREQ_EV_ELS_TMO || !cqe)
+ goto free;
+
+ /* Kill the timer we put on the request */
+ cancel_delayed_work_sync(&io_req->timeout_work);
+
+ rc = qedf_send_srr(io_req, cb_arg->offset, cb_arg->r_ctl);
+ if (rc)
+ QEDF_ERR(&(qedf->dbg_ctx), "Unable to send SRR, I/O will "
+ "abort, xid=0x%x.\n", io_req->xid);
+free:
+ kfree(cb_arg);
+ kref_put(&io_req->refcount, qedf_release_cmd);
+}
+
+static bool qedf_requeue_io_req(struct qedf_ioreq *orig_io_req)
+{
+ struct qedf_rport *fcport;
+ struct qedf_ioreq *new_io_req;
+ unsigned long flags;
+ bool rc = false;
+
+ fcport = orig_io_req->fcport;
+ if (!fcport) {
+ QEDF_ERR(NULL, "fcport is NULL.\n");
+ goto out;
+ }
+
+ if (!orig_io_req->sc_cmd) {
+ QEDF_ERR(&(fcport->qedf->dbg_ctx), "sc_cmd is NULL for "
+ "xid=0x%x.\n", orig_io_req->xid);
+ goto out;
+ }
+
+ new_io_req = qedf_alloc_cmd(fcport, QEDF_SCSI_CMD);
+ if (!new_io_req) {
+ QEDF_ERR(&(fcport->qedf->dbg_ctx), "Could not allocate new "
+ "io_req.\n");
+ goto out;
+ }
+
+ new_io_req->sc_cmd = orig_io_req->sc_cmd;
+
+ /*
+ * This keeps the sc_cmd struct from being returned to the tape
+ * driver and being requeued twice. We do need to put a reference
+ * for the original I/O request since we will not do a SCSI completion
+ * for it.
+ */
+ orig_io_req->sc_cmd = NULL;
+ kref_put(&orig_io_req->refcount, qedf_release_cmd);
+
+ spin_lock_irqsave(&fcport->rport_lock, flags);
+
+ /* kref for new command released in qedf_post_io_req on error */
+ if (qedf_post_io_req(fcport, new_io_req)) {
+ QEDF_ERR(&(fcport->qedf->dbg_ctx), "Unable to post io_req\n");
+ /* Return SQE to pool */
+ atomic_inc(&fcport->free_sqes);
+ } else {
+ QEDF_INFO(&(fcport->qedf->dbg_ctx), QEDF_LOG_ELS,
+ "Reissued SCSI command from orig_xid=0x%x on "
+ "new_xid=0x%x.\n", orig_io_req->xid, new_io_req->xid);
+ /*
+ * Abort the original I/O but do not return SCSI command as
+ * it has been reissued on another OX_ID.
+ */
+ spin_unlock_irqrestore(&fcport->rport_lock, flags);
+ qedf_initiate_abts(orig_io_req, false);
+ goto out;
+ }
+
+ spin_unlock_irqrestore(&fcport->rport_lock, flags);
+out:
+ return rc;
+}
+
+
+static void qedf_rec_compl(struct qedf_els_cb_arg *cb_arg)
+{
+ struct qedf_ioreq *orig_io_req;
+ struct qedf_ioreq *rec_req;
+ struct qedf_mp_req *mp_req;
+ struct fc_frame_header *mp_fc_hdr, *fh;
+ struct fc_frame *fp;
+ void *resp_buf, *fc_payload;
+ u32 resp_len;
+ struct fc_lport *lport;
+ struct qedf_ctx *qedf;
+ int refcount;
+ enum fc_rctl r_ctl;
+ struct fc_els_ls_rjt *rjt;
+ struct fc_els_rec_acc *acc;
+ u8 opcode;
+ u32 offset, e_stat;
+ struct scsi_cmnd *sc_cmd;
+ bool srr_needed = false;
+
+ rec_req = cb_arg->io_req;
+ qedf = rec_req->fcport->qedf;
+ lport = qedf->lport;
+
+ orig_io_req = cb_arg->aborted_io_req;
+
+ if (!orig_io_req)
+ goto out_free;
+
+ if (rec_req->event != QEDF_IOREQ_EV_ELS_TMO &&
+ rec_req->event != QEDF_IOREQ_EV_ELS_ERR_DETECT)
+ cancel_delayed_work_sync(&orig_io_req->timeout_work);
+
+ refcount = kref_read(&orig_io_req->refcount);
+ QEDF_INFO(&(qedf->dbg_ctx), QEDF_LOG_ELS, "Entered: orig_io=%p,"
+ " orig_io_xid=0x%x, rec_xid=0x%x, refcount=%d\n",
+ orig_io_req, orig_io_req->xid, rec_req->xid, refcount);
+
+ /* If a REC times out, free resources */
+ if (rec_req->event == QEDF_IOREQ_EV_ELS_TMO)
+ goto out_free;
+
+ /* Normalize response data into struct fc_frame */
+ mp_req = &(rec_req->mp_req);
+ mp_fc_hdr = &(mp_req->resp_fc_hdr);
+ resp_len = mp_req->resp_len;
+ acc = resp_buf = mp_req->resp_buf;
+
+ fp = fc_frame_alloc(lport, resp_len);
+ if (!fp) {
+ QEDF_ERR(&(qedf->dbg_ctx),
+ "fc_frame_alloc failure.\n");
+ goto out_free;
+ }
+
+ /* Copy frame header from firmware into fp */
+ fh = (struct fc_frame_header *)fc_frame_header_get(fp);
+ memcpy(fh, mp_fc_hdr, sizeof(struct fc_frame_header));
+
+ /* Copy payload from firmware into fp */
+ fc_payload = fc_frame_payload_get(fp, resp_len);
+ memcpy(fc_payload, resp_buf, resp_len);
+
+ opcode = fc_frame_payload_op(fp);
+ if (opcode == ELS_LS_RJT) {
+ rjt = fc_frame_payload_get(fp, sizeof(*rjt));
+ QEDF_INFO(&(qedf->dbg_ctx), QEDF_LOG_ELS,
+ "Received LS_RJT for REC: er_reason=0x%x, "
+ "er_explan=0x%x.\n", rjt->er_reason, rjt->er_explan);
+ /*
+ * The following response(s) mean that we need to reissue the
+ * request on another exchange. We need to do this without
+ * informing the upper layers lest it cause an application
+ * error.
+ */
+ if ((rjt->er_reason == ELS_RJT_LOGIC ||
+ rjt->er_reason == ELS_RJT_UNAB) &&
+ rjt->er_explan == ELS_EXPL_OXID_RXID) {
+ QEDF_INFO(&(qedf->dbg_ctx), QEDF_LOG_ELS,
+ "Handle CMD LOST case.\n");
+ qedf_requeue_io_req(orig_io_req);
+ }
+ } else if (opcode == ELS_LS_ACC) {
+ offset = ntohl(acc->reca_fc4value);
+ e_stat = ntohl(acc->reca_e_stat);
+ QEDF_INFO(&(qedf->dbg_ctx), QEDF_LOG_ELS,
+ "Received LS_ACC for REC: offset=0x%x, e_stat=0x%x.\n",
+ offset, e_stat);
+ if (e_stat & ESB_ST_SEQ_INIT) {
+ QEDF_INFO(&(qedf->dbg_ctx), QEDF_LOG_ELS,
+ "Target has the seq init\n");
+ goto out_free_frame;
+ }
+ sc_cmd = orig_io_req->sc_cmd;
+ if (!sc_cmd) {
+ QEDF_INFO(&(qedf->dbg_ctx), QEDF_LOG_ELS,
+ "sc_cmd is NULL for xid=0x%x.\n",
+ orig_io_req->xid);
+ goto out_free_frame;
+ }
+ /* SCSI write case */
+ if (sc_cmd->sc_data_direction == DMA_TO_DEVICE) {
+ if (offset == orig_io_req->data_xfer_len) {
+ QEDF_INFO(&(qedf->dbg_ctx), QEDF_LOG_ELS,
+ "WRITE - response lost.\n");
+ r_ctl = FC_RCTL_DD_CMD_STATUS;
+ srr_needed = true;
+ offset = 0;
+ } else {
+ QEDF_INFO(&(qedf->dbg_ctx), QEDF_LOG_ELS,
+ "WRITE - XFER_RDY/DATA lost.\n");
+ r_ctl = FC_RCTL_DD_DATA_DESC;
+ /* Use data from warning CQE instead of REC */
+ offset = orig_io_req->tx_buf_off;
+ }
+ /* SCSI read case */
+ } else {
+ if (orig_io_req->rx_buf_off ==
+ orig_io_req->data_xfer_len) {
+ QEDF_INFO(&(qedf->dbg_ctx), QEDF_LOG_ELS,
+ "READ - response lost.\n");
+ srr_needed = true;
+ r_ctl = FC_RCTL_DD_CMD_STATUS;
+ offset = 0;
+ } else {
+ QEDF_INFO(&(qedf->dbg_ctx), QEDF_LOG_ELS,
+ "READ - DATA lost.\n");
+ /*
+ * For read case we always set the offset to 0
+ * for sequence recovery task.
+ */
+ offset = 0;
+ r_ctl = FC_RCTL_DD_SOL_DATA;
+ }
+ }
+
+ if (srr_needed)
+ qedf_send_srr(orig_io_req, offset, r_ctl);
+ else
+ qedf_initiate_seq_cleanup(orig_io_req, offset, r_ctl);
+ }
+
+out_free_frame:
+ fc_frame_free(fp);
+out_free:
+ /* Put reference for original command since REC completed */
+ kref_put(&orig_io_req->refcount, qedf_release_cmd);
+ kfree(cb_arg);
+}
+
+/* Assumes kref is already held by caller */
+int qedf_send_rec(struct qedf_ioreq *orig_io_req)
+{
+
+ struct fc_els_rec rec;
+ struct qedf_rport *fcport;
+ struct fc_lport *lport;
+ struct qedf_els_cb_arg *cb_arg = NULL;
+ struct qedf_ctx *qedf;
+ uint32_t sid;
+ uint32_t r_a_tov;
+ int rc;
+
+ if (!orig_io_req) {
+ QEDF_ERR(NULL, "orig_io_req is NULL.\n");
+ return -EINVAL;
+ }
+
+ fcport = orig_io_req->fcport;
+
+ /* Check that fcport is still offloaded */
+ if (!test_bit(QEDF_RPORT_SESSION_READY, &fcport->flags)) {
+ QEDF_ERR(NULL, "fcport is no longer offloaded.\n");
+ return -EINVAL;
+ }
+
+ if (!fcport->qedf) {
+ QEDF_ERR(NULL, "fcport->qedf is NULL.\n");
+ return -EINVAL;
+ }
+
+ /* Take reference until REC command completion */
+ kref_get(&orig_io_req->refcount);
+
+ qedf = fcport->qedf;
+ lport = qedf->lport;
+ sid = fcport->sid;
+ r_a_tov = lport->r_a_tov;
+
+ memset(&rec, 0, sizeof(rec));
+
+ cb_arg = kzalloc(sizeof(struct qedf_els_cb_arg), GFP_NOIO);
+ if (!cb_arg) {
+ QEDF_ERR(&(qedf->dbg_ctx), "Unable to allocate cb_arg for "
+ "REC\n");
+ rc = -ENOMEM;
+ goto rec_err;
+ }
+
+ cb_arg->aborted_io_req = orig_io_req;
+
+ rec.rec_cmd = ELS_REC;
+ hton24(rec.rec_s_id, sid);
+ rec.rec_ox_id = htons(orig_io_req->xid);
+ rec.rec_rx_id =
+ htons(orig_io_req->task->tstorm_st_context.read_write.rx_id);
+
+ QEDF_INFO(&(qedf->dbg_ctx), QEDF_LOG_ELS, "Sending REC orig_io=%p, "
+ "orig_xid=0x%x rx_id=0x%x\n", orig_io_req,
+ orig_io_req->xid, rec.rec_rx_id);
+ rc = qedf_initiate_els(fcport, ELS_REC, &rec, sizeof(rec),
+ qedf_rec_compl, cb_arg, r_a_tov);
+
+rec_err:
+ if (rc) {
+ QEDF_ERR(&(qedf->dbg_ctx), "REC failed - release orig_io_req"
+ "=0x%x\n", orig_io_req->xid);
+ kfree(cb_arg);
+ kref_put(&orig_io_req->refcount, qedf_release_cmd);
+ }
+ return rc;
+}
--- /dev/null
+/*
+ * QLogic FCoE Offload Driver
+ * Copyright (c) 2016 Cavium Inc.
+ *
+ * This software is available under the terms of the GNU General Public License
+ * (GPL) Version 2, available from the file COPYING in the main directory of
+ * this source tree.
+ */
+#include <linux/if_ether.h>
+#include <linux/if_vlan.h>
+#include "qedf.h"
+
+extern const struct qed_fcoe_ops *qed_ops;
+/*
+ * FIP VLAN functions that will eventually move to libfcoe.
+ */
+
+void qedf_fcoe_send_vlan_req(struct qedf_ctx *qedf)
+{
+ struct sk_buff *skb;
+ char *eth_fr;
+ int fr_len;
+ struct fip_vlan *vlan;
+#define MY_FIP_ALL_FCF_MACS ((__u8[6]) { 1, 0x10, 0x18, 1, 0, 2 })
+ static u8 my_fcoe_all_fcfs[ETH_ALEN] = MY_FIP_ALL_FCF_MACS;
+
+ skb = dev_alloc_skb(sizeof(struct fip_vlan));
+ if (!skb)
+ return;
+
+ fr_len = sizeof(*vlan);
+ eth_fr = (char *)skb->data;
+ vlan = (struct fip_vlan *)eth_fr;
+
+ memset(vlan, 0, sizeof(*vlan));
+ ether_addr_copy(vlan->eth.h_source, qedf->mac);
+ ether_addr_copy(vlan->eth.h_dest, my_fcoe_all_fcfs);
+ vlan->eth.h_proto = htons(ETH_P_FIP);
+
+ vlan->fip.fip_ver = FIP_VER_ENCAPS(FIP_VER);
+ vlan->fip.fip_op = htons(FIP_OP_VLAN);
+ vlan->fip.fip_subcode = FIP_SC_VL_REQ;
+ vlan->fip.fip_dl_len = htons(sizeof(vlan->desc) / FIP_BPW);
+
+ vlan->desc.mac.fd_desc.fip_dtype = FIP_DT_MAC;
+ vlan->desc.mac.fd_desc.fip_dlen = sizeof(vlan->desc.mac) / FIP_BPW;
+ ether_addr_copy(vlan->desc.mac.fd_mac, qedf->mac);
+
+ vlan->desc.wwnn.fd_desc.fip_dtype = FIP_DT_NAME;
+ vlan->desc.wwnn.fd_desc.fip_dlen = sizeof(vlan->desc.wwnn) / FIP_BPW;
+ put_unaligned_be64(qedf->lport->wwnn, &vlan->desc.wwnn.fd_wwn);
+
+ skb_put(skb, sizeof(*vlan));
+ skb->protocol = htons(ETH_P_FIP);
+ skb_reset_mac_header(skb);
+ skb_reset_network_header(skb);
+
+ QEDF_INFO(&(qedf->dbg_ctx), QEDF_LOG_DISC, "Sending FIP VLAN "
+ "request.");
+
+ if (atomic_read(&qedf->link_state) != QEDF_LINK_UP) {
+ QEDF_WARN(&(qedf->dbg_ctx), "Cannot send vlan request "
+ "because link is not up.\n");
+
+ kfree_skb(skb);
+ return;
+ }
+ qed_ops->ll2->start_xmit(qedf->cdev, skb);
+}
+
+static void qedf_fcoe_process_vlan_resp(struct qedf_ctx *qedf,
+ struct sk_buff *skb)
+{
+ struct fip_header *fiph;
+ struct fip_desc *desc;
+ u16 vid = 0;
+ ssize_t rlen;
+ size_t dlen;
+
+ fiph = (struct fip_header *)(((void *)skb->data) + 2 * ETH_ALEN + 2);
+
+ rlen = ntohs(fiph->fip_dl_len) * 4;
+ desc = (struct fip_desc *)(fiph + 1);
+ while (rlen > 0) {
+ dlen = desc->fip_dlen * FIP_BPW;
+ switch (desc->fip_dtype) {
+ case FIP_DT_VLAN:
+ vid = ntohs(((struct fip_vlan_desc *)desc)->fd_vlan);
+ break;
+ }
+ desc = (struct fip_desc *)((char *)desc + dlen);
+ rlen -= dlen;
+ }
+
+ QEDF_INFO(&(qedf->dbg_ctx), QEDF_LOG_DISC, "VLAN response, "
+ "vid=0x%x.\n", vid);
+
+ if (vid > 0 && qedf->vlan_id != vid) {
+ qedf_set_vlan_id(qedf, vid);
+
+ /* Inform waiter that it's ok to call fcoe_ctlr_link up() */
+ complete(&qedf->fipvlan_compl);
+ }
+}
+
+void qedf_fip_send(struct fcoe_ctlr *fip, struct sk_buff *skb)
+{
+ struct qedf_ctx *qedf = container_of(fip, struct qedf_ctx, ctlr);
+ struct ethhdr *eth_hdr;
+ struct vlan_ethhdr *vlan_hdr;
+ struct fip_header *fiph;
+ u16 op, vlan_tci = 0;
+ u8 sub;
+
+ if (!test_bit(QEDF_LL2_STARTED, &qedf->flags)) {
+ QEDF_WARN(&(qedf->dbg_ctx), "LL2 not started\n");
+ kfree_skb(skb);
+ return;
+ }
+
+ fiph = (struct fip_header *) ((void *)skb->data + 2 * ETH_ALEN + 2);
+ eth_hdr = (struct ethhdr *)skb_mac_header(skb);
+ op = ntohs(fiph->fip_op);
+ sub = fiph->fip_subcode;
+
+ if (!qedf->vlan_hw_insert) {
+ vlan_hdr = (struct vlan_ethhdr *)skb_push(skb, sizeof(*vlan_hdr)
+ - sizeof(*eth_hdr));
+ memcpy(vlan_hdr, eth_hdr, 2 * ETH_ALEN);
+ vlan_hdr->h_vlan_proto = htons(ETH_P_8021Q);
+ vlan_hdr->h_vlan_encapsulated_proto = eth_hdr->h_proto;
+ vlan_hdr->h_vlan_TCI = vlan_tci = htons(qedf->vlan_id);
+ }
+
+ /* Update eth_hdr since we added a VLAN tag */
+ eth_hdr = (struct ethhdr *)skb_mac_header(skb);
+
+ QEDF_INFO(&(qedf->dbg_ctx), QEDF_LOG_LL2, "FIP frame send: "
+ "dest=%pM op=%x sub=%x vlan=%04x.", eth_hdr->h_dest, op, sub,
+ ntohs(vlan_tci));
+ if (qedf_dump_frames)
+ print_hex_dump(KERN_WARNING, "fip ", DUMP_PREFIX_OFFSET, 16, 1,
+ skb->data, skb->len, false);
+
+ qed_ops->ll2->start_xmit(qedf->cdev, skb);
+}
+
+/* Process incoming FIP frames. */
+void qedf_fip_recv(struct qedf_ctx *qedf, struct sk_buff *skb)
+{
+ struct ethhdr *eth_hdr;
+ struct fip_header *fiph;
+ struct fip_desc *desc;
+ struct fip_mac_desc *mp;
+ struct fip_wwn_desc *wp;
+ struct fip_vn_desc *vp;
+ size_t rlen, dlen;
+ uint32_t cvl_port_id;
+ __u8 cvl_mac[ETH_ALEN];
+ u16 op;
+ u8 sub;
+
+ eth_hdr = (struct ethhdr *)skb_mac_header(skb);
+ fiph = (struct fip_header *) ((void *)skb->data + 2 * ETH_ALEN + 2);
+ op = ntohs(fiph->fip_op);
+ sub = fiph->fip_subcode;
+
+ QEDF_INFO(&(qedf->dbg_ctx), QEDF_LOG_LL2, "FIP frame received: "
+ "skb=%p fiph=%p source=%pM op=%x sub=%x", skb, fiph,
+ eth_hdr->h_source, op, sub);
+ if (qedf_dump_frames)
+ print_hex_dump(KERN_WARNING, "fip ", DUMP_PREFIX_OFFSET, 16, 1,
+ skb->data, skb->len, false);
+
+ /* Handle FIP VLAN resp in the driver */
+ if (op == FIP_OP_VLAN && sub == FIP_SC_VL_NOTE) {
+ qedf_fcoe_process_vlan_resp(qedf, skb);
+ qedf->vlan_hw_insert = 0;
+ kfree_skb(skb);
+ } else if (op == FIP_OP_CTRL && sub == FIP_SC_CLR_VLINK) {
+ QEDF_INFO(&(qedf->dbg_ctx), QEDF_LOG_DISC, "Clear virtual "
+ "link received.\n");
+
+ /* Check that an FCF has been selected by fcoe */
+ if (qedf->ctlr.sel_fcf == NULL) {
+ QEDF_INFO(&(qedf->dbg_ctx), QEDF_LOG_DISC,
+ "Dropping CVL since FCF has not been selected "
+ "yet.");
+ return;
+ }
+
+ cvl_port_id = 0;
+ memset(cvl_mac, 0, ETH_ALEN);
+ /*
+ * We need to loop through the CVL descriptors to determine
+ * if we want to reset the fcoe link
+ */
+ rlen = ntohs(fiph->fip_dl_len) * FIP_BPW;
+ desc = (struct fip_desc *)(fiph + 1);
+ while (rlen >= sizeof(*desc)) {
+ dlen = desc->fip_dlen * FIP_BPW;
+ switch (desc->fip_dtype) {
+ case FIP_DT_MAC:
+ mp = (struct fip_mac_desc *)desc;
+ QEDF_INFO(&(qedf->dbg_ctx), QEDF_LOG_LL2,
+ "fd_mac=%pM.\n", __func__, mp->fd_mac);
+ ether_addr_copy(cvl_mac, mp->fd_mac);
+ break;
+ case FIP_DT_NAME:
+ wp = (struct fip_wwn_desc *)desc;
+ QEDF_INFO(&(qedf->dbg_ctx), QEDF_LOG_LL2,
+ "fc_wwpn=%016llx.\n",
+ get_unaligned_be64(&wp->fd_wwn));
+ break;
+ case FIP_DT_VN_ID:
+ vp = (struct fip_vn_desc *)desc;
+ QEDF_INFO(&(qedf->dbg_ctx), QEDF_LOG_LL2,
+ "fd_fc_id=%x.\n", ntoh24(vp->fd_fc_id));
+ cvl_port_id = ntoh24(vp->fd_fc_id);
+ break;
+ default:
+ /* Ignore anything else */
+ break;
+ }
+ desc = (struct fip_desc *)((char *)desc + dlen);
+ rlen -= dlen;
+ }
+
+ QEDF_INFO(&(qedf->dbg_ctx), QEDF_LOG_LL2,
+ "cvl_port_id=%06x cvl_mac=%pM.\n", cvl_port_id,
+ cvl_mac);
+ if (cvl_port_id == qedf->lport->port_id &&
+ ether_addr_equal(cvl_mac,
+ qedf->ctlr.sel_fcf->fcf_mac)) {
+ fcoe_ctlr_link_down(&qedf->ctlr);
+ qedf_wait_for_upload(qedf);
+ fcoe_ctlr_link_up(&qedf->ctlr);
+ }
+ kfree_skb(skb);
+ } else {
+ /* Everything else is handled by libfcoe */
+ __skb_pull(skb, ETH_HLEN);
+ fcoe_ctlr_recv(&qedf->ctlr, skb);
+ }
+}
+
+void qedf_update_src_mac(struct fc_lport *lport, u8 *addr)
+{
+ struct qedf_ctx *qedf = lport_priv(lport);
+
+ QEDF_INFO(&(qedf->dbg_ctx), QEDF_LOG_DISC,
+ "Setting data_src_addr=%pM.\n", addr);
+ ether_addr_copy(qedf->data_src_addr, addr);
+}
+
+u8 *qedf_get_src_mac(struct fc_lport *lport)
+{
+ u8 mac[ETH_ALEN];
+ u8 port_id[3];
+ struct qedf_ctx *qedf = lport_priv(lport);
+
+ /* We need to use the lport port_id to create the data_src_addr */
+ if (is_zero_ether_addr(qedf->data_src_addr)) {
+ hton24(port_id, lport->port_id);
+ fc_fcoe_set_mac(mac, port_id);
+ qedf->ctlr.update_mac(lport, mac);
+ }
+ return qedf->data_src_addr;
+}
--- /dev/null
+/*
+ * QLogic FCoE Offload Driver
+ * Copyright (c) 2016 Cavium Inc.
+ *
+ * This software is available under the terms of the GNU General Public License
+ * (GPL) Version 2, available from the file COPYING in the main directory of
+ * this source tree.
+ */
+#ifndef __QEDF_HSI__
+#define __QEDF_HSI__
+/*
+ * Add include to common target
+ */
+#include <linux/qed/common_hsi.h>
+
+/*
+ * Add include to common storage target
+ */
+#include <linux/qed/storage_common.h>
+
+/*
+ * Add include to common fcoe target for both eCore and protocol driver
+ */
+#include <linux/qed/fcoe_common.h>
+
+
+/*
+ * FCoE CQ element ABTS information
+ */
+struct fcoe_abts_info {
+ u8 r_ctl /* R_CTL in the ABTS response frame */;
+ u8 reserved0;
+ __le16 rx_id;
+ __le32 reserved2[2];
+ __le32 fc_payload[3] /* ABTS FC payload response frame */;
+};
+
+
+/*
+ * FCoE class type
+ */
+enum fcoe_class_type {
+ FCOE_TASK_CLASS_TYPE_3,
+ FCOE_TASK_CLASS_TYPE_2,
+ MAX_FCOE_CLASS_TYPE
+};
+
+
+/*
+ * FCoE CMDQ element control information
+ */
+struct fcoe_cmdqe_control {
+ __le16 conn_id;
+ u8 num_additional_cmdqes;
+ u8 cmdType;
+ /* true for ABTS request cmdqe. used in Target mode */
+#define FCOE_CMDQE_CONTROL_ABTSREQCMD_MASK 0x1
+#define FCOE_CMDQE_CONTROL_ABTSREQCMD_SHIFT 0
+#define FCOE_CMDQE_CONTROL_RESERVED1_MASK 0x7F
+#define FCOE_CMDQE_CONTROL_RESERVED1_SHIFT 1
+ u8 reserved2[4];
+};
+
+/*
+ * FCoE control + payload CMDQ element
+ */
+struct fcoe_cmdqe {
+ struct fcoe_cmdqe_control hdr;
+ u8 fc_header[24];
+ __le32 fcp_cmd_payload[8];
+};
+
+
+
+/*
+ * FCP RSP flags
+ */
+struct fcoe_fcp_rsp_flags {
+ u8 flags;
+#define FCOE_FCP_RSP_FLAGS_FCP_RSP_LEN_VALID_MASK 0x1
+#define FCOE_FCP_RSP_FLAGS_FCP_RSP_LEN_VALID_SHIFT 0
+#define FCOE_FCP_RSP_FLAGS_FCP_SNS_LEN_VALID_MASK 0x1
+#define FCOE_FCP_RSP_FLAGS_FCP_SNS_LEN_VALID_SHIFT 1
+#define FCOE_FCP_RSP_FLAGS_FCP_RESID_OVER_MASK 0x1
+#define FCOE_FCP_RSP_FLAGS_FCP_RESID_OVER_SHIFT 2
+#define FCOE_FCP_RSP_FLAGS_FCP_RESID_UNDER_MASK 0x1
+#define FCOE_FCP_RSP_FLAGS_FCP_RESID_UNDER_SHIFT 3
+#define FCOE_FCP_RSP_FLAGS_FCP_CONF_REQ_MASK 0x1
+#define FCOE_FCP_RSP_FLAGS_FCP_CONF_REQ_SHIFT 4
+#define FCOE_FCP_RSP_FLAGS_FCP_BIDI_FLAGS_MASK 0x7
+#define FCOE_FCP_RSP_FLAGS_FCP_BIDI_FLAGS_SHIFT 5
+};
+
+/*
+ * FCoE CQ element response information
+ */
+struct fcoe_cqe_rsp_info {
+ struct fcoe_fcp_rsp_flags rsp_flags;
+ u8 scsi_status_code;
+ __le16 retry_delay_timer;
+ __le32 fcp_resid;
+ __le32 fcp_sns_len;
+ __le32 fcp_rsp_len;
+ __le16 rx_id;
+ u8 fw_error_flags;
+#define FCOE_CQE_RSP_INFO_FW_UNDERRUN_MASK 0x1 /* FW detected underrun */
+#define FCOE_CQE_RSP_INFO_FW_UNDERRUN_SHIFT 0
+#define FCOE_CQE_RSP_INFO_RESREVED_MASK 0x7F
+#define FCOE_CQE_RSP_INFO_RESREVED_SHIFT 1
+ u8 reserved;
+ __le32 fw_residual /* Residual bytes calculated by FW */;
+};
+
+/*
+ * FCoE CQ element Target completion information
+ */
+struct fcoe_cqe_target_info {
+ __le16 rx_id;
+ __le16 reserved0;
+ __le32 reserved1[5];
+};
+
+/*
+ * FCoE error/warning reporting entry
+ */
+struct fcoe_err_report_entry {
+ __le32 err_warn_bitmap_lo /* Error bitmap lower 32 bits */;
+ __le32 err_warn_bitmap_hi /* Error bitmap higher 32 bits */;
+ /* Buffer offset the beginning of the Sequence last transmitted */
+ __le32 tx_buf_off;
+ /* Buffer offset from the beginning of the Sequence last received */
+ __le32 rx_buf_off;
+ __le16 rx_id /* RX_ID of the associated task */;
+ __le16 reserved1;
+ __le32 reserved2;
+};
+
+/*
+ * FCoE CQ element middle path information
+ */
+struct fcoe_cqe_midpath_info {
+ __le32 data_placement_size;
+ __le16 rx_id;
+ __le16 reserved0;
+ __le32 reserved1[4];
+};
+
+/*
+ * FCoE CQ element unsolicited information
+ */
+struct fcoe_unsolic_info {
+ /* BD information: Physical address and opaque data */
+ struct scsi_bd bd_info;
+ __le16 conn_id /* Connection ID the frame is associated to */;
+ __le16 pkt_len /* Packet length */;
+ u8 reserved1[4];
+};
+
+/*
+ * FCoE warning reporting entry
+ */
+struct fcoe_warning_report_entry {
+ /* BD information: Physical address and opaque data */
+ struct scsi_bd bd_info;
+ /* Buffer offset the beginning of the Sequence last transmitted */
+ __le32 buf_off;
+ __le16 rx_id /* RX_ID of the associated task */;
+ __le16 reserved1;
+};
+
+/*
+ * FCoE CQ element information
+ */
+union fcoe_cqe_info {
+ struct fcoe_cqe_rsp_info rsp_info /* Response completion information */;
+ /* Target completion information */
+ struct fcoe_cqe_target_info target_info;
+ /* Error completion information */
+ struct fcoe_err_report_entry err_info;
+ struct fcoe_abts_info abts_info /* ABTS completion information */;
+ /* Middle path completion information */
+ struct fcoe_cqe_midpath_info midpath_info;
+ /* Unsolicited packet completion information */
+ struct fcoe_unsolic_info unsolic_info;
+ /* Warning completion information (Rec Tov expiration) */
+ struct fcoe_warning_report_entry warn_info;
+};
+
+/*
+ * FCoE CQ element
+ */
+struct fcoe_cqe {
+ __le32 cqe_data;
+ /* The task identifier (OX_ID) to be completed */
+#define FCOE_CQE_TASK_ID_MASK 0xFFFF
+#define FCOE_CQE_TASK_ID_SHIFT 0
+ /*
+ * The CQE type: 0x0 Indicating on a pending work request completion.
+ * 0x1 - Indicating on an unsolicited event notification. use enum
+ * fcoe_cqe_type (use enum fcoe_cqe_type)
+ */
+#define FCOE_CQE_CQE_TYPE_MASK 0xF
+#define FCOE_CQE_CQE_TYPE_SHIFT 16
+#define FCOE_CQE_RESERVED0_MASK 0xFFF
+#define FCOE_CQE_RESERVED0_SHIFT 20
+ __le16 reserved1;
+ __le16 fw_cq_prod;
+ union fcoe_cqe_info cqe_info;
+};
+
+/*
+ * FCoE CQE type
+ */
+enum fcoe_cqe_type {
+ /* solicited response on a R/W or middle-path SQE */
+ FCOE_GOOD_COMPLETION_CQE_TYPE,
+ FCOE_UNSOLIC_CQE_TYPE /* unsolicited packet, RQ consumed */,
+ FCOE_ERROR_DETECTION_CQE_TYPE /* timer expiration, validation error */,
+ FCOE_WARNING_CQE_TYPE /* rec_tov or rr_tov timer expiration */,
+ FCOE_EXCH_CLEANUP_CQE_TYPE /* task cleanup completed */,
+ FCOE_ABTS_CQE_TYPE /* ABTS received and task cleaned */,
+ FCOE_DUMMY_CQE_TYPE /* just increment SQ CONS */,
+ /* Task was completed wight after sending a pkt to the target */
+ FCOE_LOCAL_COMP_CQE_TYPE,
+ MAX_FCOE_CQE_TYPE
+};
+
+
+/*
+ * FCoE device type
+ */
+enum fcoe_device_type {
+ FCOE_TASK_DEV_TYPE_DISK,
+ FCOE_TASK_DEV_TYPE_TAPE,
+ MAX_FCOE_DEVICE_TYPE
+};
+
+
+
+
+/*
+ * FCoE fast path error codes
+ */
+enum fcoe_fp_error_warning_code {
+ FCOE_ERROR_CODE_XFER_OOO_RO /* XFER error codes */,
+ FCOE_ERROR_CODE_XFER_RO_NOT_ALIGNED,
+ FCOE_ERROR_CODE_XFER_NULL_BURST_LEN,
+ FCOE_ERROR_CODE_XFER_RO_GREATER_THAN_DATA2TRNS,
+ FCOE_ERROR_CODE_XFER_INVALID_PAYLOAD_SIZE,
+ FCOE_ERROR_CODE_XFER_TASK_TYPE_NOT_WRITE,
+ FCOE_ERROR_CODE_XFER_PEND_XFER_SET,
+ FCOE_ERROR_CODE_XFER_OPENED_SEQ,
+ FCOE_ERROR_CODE_XFER_FCTL,
+ FCOE_ERROR_CODE_FCP_RSP_BIDI_FLAGS_SET /* FCP RSP error codes */,
+ FCOE_ERROR_CODE_FCP_RSP_INVALID_LENGTH_FIELD,
+ FCOE_ERROR_CODE_FCP_RSP_INVALID_SNS_FIELD,
+ FCOE_ERROR_CODE_FCP_RSP_INVALID_PAYLOAD_SIZE,
+ FCOE_ERROR_CODE_FCP_RSP_PEND_XFER_SET,
+ FCOE_ERROR_CODE_FCP_RSP_OPENED_SEQ,
+ FCOE_ERROR_CODE_FCP_RSP_FCTL,
+ FCOE_ERROR_CODE_FCP_RSP_LAST_SEQ_RESET,
+ FCOE_ERROR_CODE_FCP_RSP_CONF_REQ_NOT_SUPPORTED_YET,
+ FCOE_ERROR_CODE_DATA_OOO_RO /* FCP DATA error codes */,
+ FCOE_ERROR_CODE_DATA_EXCEEDS_DEFINED_MAX_FRAME_SIZE,
+ FCOE_ERROR_CODE_DATA_EXCEEDS_DATA2TRNS,
+ FCOE_ERROR_CODE_DATA_SOFI3_SEQ_ACTIVE_SET,
+ FCOE_ERROR_CODE_DATA_SOFN_SEQ_ACTIVE_RESET,
+ FCOE_ERROR_CODE_DATA_EOFN_END_SEQ_SET,
+ FCOE_ERROR_CODE_DATA_EOFT_END_SEQ_RESET,
+ FCOE_ERROR_CODE_DATA_TASK_TYPE_NOT_READ,
+ FCOE_ERROR_CODE_DATA_FCTL_INITIATIR,
+ FCOE_ERROR_CODE_MIDPATH_INVALID_TYPE /* Middle path error codes */,
+ FCOE_ERROR_CODE_MIDPATH_SOFI3_SEQ_ACTIVE_SET,
+ FCOE_ERROR_CODE_MIDPATH_SOFN_SEQ_ACTIVE_RESET,
+ FCOE_ERROR_CODE_MIDPATH_EOFN_END_SEQ_SET,
+ FCOE_ERROR_CODE_MIDPATH_EOFT_END_SEQ_RESET,
+ FCOE_ERROR_CODE_MIDPATH_REPLY_FCTL,
+ FCOE_ERROR_CODE_MIDPATH_INVALID_REPLY,
+ FCOE_ERROR_CODE_MIDPATH_ELS_REPLY_RCTL,
+ FCOE_ERROR_CODE_COMMON_MIDDLE_FRAME_WITH_PAD /* Common error codes */,
+ FCOE_ERROR_CODE_COMMON_SEQ_INIT_IN_TCE,
+ FCOE_ERROR_CODE_COMMON_FC_HDR_RX_ID_MISMATCH,
+ FCOE_ERROR_CODE_COMMON_INCORRECT_SEQ_CNT,
+ FCOE_ERROR_CODE_COMMON_DATA_FC_HDR_FCP_TYPE_MISMATCH,
+ FCOE_ERROR_CODE_COMMON_DATA_NO_MORE_SGES,
+ FCOE_ERROR_CODE_COMMON_OPTIONAL_FC_HDR,
+ FCOE_ERROR_CODE_COMMON_READ_TCE_OX_ID_TOO_BIG,
+ FCOE_ERROR_CODE_COMMON_DATA_WAS_NOT_TRANSMITTED,
+ FCOE_ERROR_CODE_COMMON_TASK_DDF_RCTL_INFO_FIELD,
+ FCOE_ERROR_CODE_COMMON_TASK_INVALID_RCTL,
+ FCOE_ERROR_CODE_COMMON_TASK_RCTL_GENERAL_MISMATCH,
+ FCOE_ERROR_CODE_E_D_TOV_TIMER_EXPIRATION /* Timer error codes */,
+ FCOE_WARNING_CODE_REC_TOV_TIMER_EXPIRATION /* Timer error codes */,
+ FCOE_ERROR_CODE_RR_TOV_TIMER_EXPIRATION /* Timer error codes */,
+ /* ABTSrsp pckt arrived unexpected */
+ FCOE_ERROR_CODE_ABTS_REPLY_UNEXPECTED,
+ FCOE_ERROR_CODE_TARGET_MODE_FCP_RSP,
+ FCOE_ERROR_CODE_TARGET_MODE_FCP_XFER,
+ FCOE_ERROR_CODE_TARGET_MODE_DATA_TASK_TYPE_NOT_WRITE,
+ FCOE_ERROR_CODE_DATA_FCTL_TARGET,
+ FCOE_ERROR_CODE_TARGET_DATA_SIZE_NO_MATCH_XFER,
+ FCOE_ERROR_CODE_TARGET_DIF_CRC_CHECKSUM_ERROR,
+ FCOE_ERROR_CODE_TARGET_DIF_REF_TAG_ERROR,
+ FCOE_ERROR_CODE_TARGET_DIF_APP_TAG_ERROR,
+ MAX_FCOE_FP_ERROR_WARNING_CODE
+};
+
+
+/*
+ * FCoE RESPQ element
+ */
+struct fcoe_respqe {
+ __le16 ox_id /* OX_ID that is located in the FCP_RSP FC header */;
+ __le16 rx_id /* RX_ID that is located in the FCP_RSP FC header */;
+ __le32 additional_info;
+/* PARAM that is located in the FCP_RSP FC header */
+#define FCOE_RESPQE_PARAM_MASK 0xFFFFFF
+#define FCOE_RESPQE_PARAM_SHIFT 0
+/* Indication whther its Target-auto-rsp mode or not */
+#define FCOE_RESPQE_TARGET_AUTO_RSP_MASK 0xFF
+#define FCOE_RESPQE_TARGET_AUTO_RSP_SHIFT 24
+};
+
+
+/*
+ * FCoE slow path error codes
+ */
+enum fcoe_sp_error_code {
+ /* Error codes for Error Reporting in slow path flows */
+ FCOE_ERROR_CODE_SLOW_PATH_TOO_MANY_FUNCS,
+ FCOE_ERROR_SLOW_PATH_CODE_NO_LICENSE,
+ MAX_FCOE_SP_ERROR_CODE
+};
+
+
+/*
+ * FCoE SQE request type
+ */
+enum fcoe_sqe_request_type {
+ SEND_FCOE_CMD,
+ SEND_FCOE_MIDPATH,
+ SEND_FCOE_ABTS_REQUEST,
+ FCOE_EXCHANGE_CLEANUP,
+ FCOE_SEQUENCE_RECOVERY,
+ SEND_FCOE_XFER_RDY,
+ SEND_FCOE_RSP,
+ SEND_FCOE_RSP_WITH_SENSE_DATA,
+ SEND_FCOE_TARGET_DATA,
+ SEND_FCOE_INITIATOR_DATA,
+ /*
+ * Xfer Continuation (==1) ready to be sent. Previous XFERs data
+ * received successfully.
+ */
+ SEND_FCOE_XFER_CONTINUATION_RDY,
+ SEND_FCOE_TARGET_ABTS_RSP,
+ MAX_FCOE_SQE_REQUEST_TYPE
+};
+
+
+/*
+ * FCoE task TX state
+ */
+enum fcoe_task_tx_state {
+ /* Initiate state after driver has initialized the task */
+ FCOE_TASK_TX_STATE_NORMAL,
+ /* Updated by TX path after complete transmitting unsolicited packet */
+ FCOE_TASK_TX_STATE_UNSOLICITED_COMPLETED,
+ /*
+ * Updated by TX path after start processing the task requesting the
+ * cleanup/abort operation
+ */
+ FCOE_TASK_TX_STATE_CLEAN_REQ,
+ FCOE_TASK_TX_STATE_ABTS /* Updated by TX path during abort procedure */,
+ /* Updated by TX path during exchange cleanup procedure */
+ FCOE_TASK_TX_STATE_EXCLEANUP,
+ /*
+ * Updated by TX path during exchange cleanup continuation task
+ * procedure
+ */
+ FCOE_TASK_TX_STATE_EXCLEANUP_TARGET_WRITE_CONT,
+ /* Updated by TX path during exchange cleanup first xfer procedure */
+ FCOE_TASK_TX_STATE_EXCLEANUP_TARGET_WRITE,
+ /* Updated by TX path during exchange cleanup read task in Target */
+ FCOE_TASK_TX_STATE_EXCLEANUP_TARGET_READ_OR_RSP,
+ /* Updated by TX path during target exchange cleanup procedure */
+ FCOE_TASK_TX_STATE_EXCLEANUP_TARGET_WRITE_LAST_CYCLE,
+ /* Updated by TX path during sequence recovery procedure */
+ FCOE_TASK_TX_STATE_SEQRECOVERY,
+ MAX_FCOE_TASK_TX_STATE
+};
+
+
+/*
+ * FCoE task type
+ */
+enum fcoe_task_type {
+ FCOE_TASK_TYPE_WRITE_INITIATOR,
+ FCOE_TASK_TYPE_READ_INITIATOR,
+ FCOE_TASK_TYPE_MIDPATH,
+ FCOE_TASK_TYPE_UNSOLICITED,
+ FCOE_TASK_TYPE_ABTS,
+ FCOE_TASK_TYPE_EXCHANGE_CLEANUP,
+ FCOE_TASK_TYPE_SEQUENCE_CLEANUP,
+ FCOE_TASK_TYPE_WRITE_TARGET,
+ FCOE_TASK_TYPE_READ_TARGET,
+ FCOE_TASK_TYPE_RSP,
+ FCOE_TASK_TYPE_RSP_SENSE_DATA,
+ FCOE_TASK_TYPE_ABTS_TARGET,
+ FCOE_TASK_TYPE_ENUM_SIZE,
+ MAX_FCOE_TASK_TYPE
+};
+
+struct scsi_glbl_queue_entry {
+ /* Start physical address for the RQ (receive queue) PBL. */
+ struct regpair rq_pbl_addr;
+ /* Start physical address for the CQ (completion queue) PBL. */
+ struct regpair cq_pbl_addr;
+ /* Start physical address for the CMDQ (command queue) PBL. */
+ struct regpair cmdq_pbl_addr;
+};
+
+#endif /* __QEDF_HSI__ */
--- /dev/null
+/*
+ * QLogic FCoE Offload Driver
+ * Copyright (c) 2016 Cavium Inc.
+ *
+ * This software is available under the terms of the GNU General Public License
+ * (GPL) Version 2, available from the file COPYING in the main directory of
+ * this source tree.
+ */
+#include <linux/spinlock.h>
+#include <linux/vmalloc.h>
+#include "qedf.h"
+#include <scsi/scsi_tcq.h>
+
+void qedf_cmd_timer_set(struct qedf_ctx *qedf, struct qedf_ioreq *io_req,
+ unsigned int timer_msec)
+{
+ queue_delayed_work(qedf->timer_work_queue, &io_req->timeout_work,
+ msecs_to_jiffies(timer_msec));
+}
+
+static void qedf_cmd_timeout(struct work_struct *work)
+{
+
+ struct qedf_ioreq *io_req =
+ container_of(work, struct qedf_ioreq, timeout_work.work);
+ struct qedf_ctx *qedf = io_req->fcport->qedf;
+ struct qedf_rport *fcport = io_req->fcport;
+ u8 op = 0;
+
+ switch (io_req->cmd_type) {
+ case QEDF_ABTS:
+ QEDF_ERR((&qedf->dbg_ctx), "ABTS timeout, xid=0x%x.\n",
+ io_req->xid);
+ /* Cleanup timed out ABTS */
+ qedf_initiate_cleanup(io_req, true);
+ complete(&io_req->abts_done);
+
+ /*
+ * Need to call kref_put for reference taken when initiate_abts
+ * was called since abts_compl won't be called now that we've
+ * cleaned up the task.
+ */
+ kref_put(&io_req->refcount, qedf_release_cmd);
+
+ /*
+ * Now that the original I/O and the ABTS are complete see
+ * if we need to reconnect to the target.
+ */
+ qedf_restart_rport(fcport);
+ break;
+ case QEDF_ELS:
+ kref_get(&io_req->refcount);
+ /*
+ * Don't attempt to clean an ELS timeout as any subseqeunt
+ * ABTS or cleanup requests just hang. For now just free
+ * the resources of the original I/O and the RRQ
+ */
+ QEDF_ERR(&(qedf->dbg_ctx), "ELS timeout, xid=0x%x.\n",
+ io_req->xid);
+ io_req->event = QEDF_IOREQ_EV_ELS_TMO;
+ /* Call callback function to complete command */
+ if (io_req->cb_func && io_req->cb_arg) {
+ op = io_req->cb_arg->op;
+ io_req->cb_func(io_req->cb_arg);
+ io_req->cb_arg = NULL;
+ }
+ qedf_initiate_cleanup(io_req, true);
+ kref_put(&io_req->refcount, qedf_release_cmd);
+ break;
+ case QEDF_SEQ_CLEANUP:
+ QEDF_ERR(&(qedf->dbg_ctx), "Sequence cleanup timeout, "
+ "xid=0x%x.\n", io_req->xid);
+ qedf_initiate_cleanup(io_req, true);
+ io_req->event = QEDF_IOREQ_EV_ELS_TMO;
+ qedf_process_seq_cleanup_compl(qedf, NULL, io_req);
+ break;
+ default:
+ break;
+ }
+}
+
+void qedf_cmd_mgr_free(struct qedf_cmd_mgr *cmgr)
+{
+ struct io_bdt *bdt_info;
+ struct qedf_ctx *qedf = cmgr->qedf;
+ size_t bd_tbl_sz;
+ u16 min_xid = QEDF_MIN_XID;
+ u16 max_xid = (FCOE_PARAMS_NUM_TASKS - 1);
+ int num_ios;
+ int i;
+ struct qedf_ioreq *io_req;
+
+ num_ios = max_xid - min_xid + 1;
+
+ /* Free fcoe_bdt_ctx structures */
+ if (!cmgr->io_bdt_pool)
+ goto free_cmd_pool;
+
+ bd_tbl_sz = QEDF_MAX_BDS_PER_CMD * sizeof(struct fcoe_sge);
+ for (i = 0; i < num_ios; i++) {
+ bdt_info = cmgr->io_bdt_pool[i];
+ if (bdt_info->bd_tbl) {
+ dma_free_coherent(&qedf->pdev->dev, bd_tbl_sz,
+ bdt_info->bd_tbl, bdt_info->bd_tbl_dma);
+ bdt_info->bd_tbl = NULL;
+ }
+ }
+
+ /* Destroy io_bdt pool */
+ for (i = 0; i < num_ios; i++) {
+ kfree(cmgr->io_bdt_pool[i]);
+ cmgr->io_bdt_pool[i] = NULL;
+ }
+
+ kfree(cmgr->io_bdt_pool);
+ cmgr->io_bdt_pool = NULL;
+
+free_cmd_pool:
+
+ for (i = 0; i < num_ios; i++) {
+ io_req = &cmgr->cmds[i];
+ /* Make sure we free per command sense buffer */
+ if (io_req->sense_buffer)
+ dma_free_coherent(&qedf->pdev->dev,
+ QEDF_SCSI_SENSE_BUFFERSIZE, io_req->sense_buffer,
+ io_req->sense_buffer_dma);
+ cancel_delayed_work_sync(&io_req->rrq_work);
+ }
+
+ /* Free command manager itself */
+ vfree(cmgr);
+}
+
+static void qedf_handle_rrq(struct work_struct *work)
+{
+ struct qedf_ioreq *io_req =
+ container_of(work, struct qedf_ioreq, rrq_work.work);
+
+ qedf_send_rrq(io_req);
+
+}
+
+struct qedf_cmd_mgr *qedf_cmd_mgr_alloc(struct qedf_ctx *qedf)
+{
+ struct qedf_cmd_mgr *cmgr;
+ struct io_bdt *bdt_info;
+ struct qedf_ioreq *io_req;
+ u16 xid;
+ int i;
+ int num_ios;
+ u16 min_xid = QEDF_MIN_XID;
+ u16 max_xid = (FCOE_PARAMS_NUM_TASKS - 1);
+
+ /* Make sure num_queues is already set before calling this function */
+ if (!qedf->num_queues) {
+ QEDF_ERR(&(qedf->dbg_ctx), "num_queues is not set.\n");
+ return NULL;
+ }
+
+ if (max_xid <= min_xid || max_xid == FC_XID_UNKNOWN) {
+ QEDF_WARN(&(qedf->dbg_ctx), "Invalid min_xid 0x%x and "
+ "max_xid 0x%x.\n", min_xid, max_xid);
+ return NULL;
+ }
+
+ QEDF_INFO(&(qedf->dbg_ctx), QEDF_LOG_DISC, "min xid 0x%x, max xid "
+ "0x%x.\n", min_xid, max_xid);
+
+ num_ios = max_xid - min_xid + 1;
+
+ cmgr = vzalloc(sizeof(struct qedf_cmd_mgr));
+ if (!cmgr) {
+ QEDF_WARN(&(qedf->dbg_ctx), "Failed to alloc cmd mgr.\n");
+ return NULL;
+ }
+
+ cmgr->qedf = qedf;
+ spin_lock_init(&cmgr->lock);
+
+ /*
+ * Initialize list of qedf_ioreq.
+ */
+ xid = QEDF_MIN_XID;
+
+ for (i = 0; i < num_ios; i++) {
+ io_req = &cmgr->cmds[i];
+ INIT_DELAYED_WORK(&io_req->timeout_work, qedf_cmd_timeout);
+
+ io_req->xid = xid++;
+
+ INIT_DELAYED_WORK(&io_req->rrq_work, qedf_handle_rrq);
+
+ /* Allocate DMA memory to hold sense buffer */
+ io_req->sense_buffer = dma_alloc_coherent(&qedf->pdev->dev,
+ QEDF_SCSI_SENSE_BUFFERSIZE, &io_req->sense_buffer_dma,
+ GFP_KERNEL);
+ if (!io_req->sense_buffer)
+ goto mem_err;
+ }
+
+ /* Allocate pool of io_bdts - one for each qedf_ioreq */
+ cmgr->io_bdt_pool = kmalloc_array(num_ios, sizeof(struct io_bdt *),
+ GFP_KERNEL);
+
+ if (!cmgr->io_bdt_pool) {
+ QEDF_WARN(&(qedf->dbg_ctx), "Failed to alloc io_bdt_pool.\n");
+ goto mem_err;
+ }
+
+ for (i = 0; i < num_ios; i++) {
+ cmgr->io_bdt_pool[i] = kmalloc(sizeof(struct io_bdt),
+ GFP_KERNEL);
+ if (!cmgr->io_bdt_pool[i]) {
+ QEDF_WARN(&(qedf->dbg_ctx), "Failed to alloc "
+ "io_bdt_pool[%d].\n", i);
+ goto mem_err;
+ }
+ }
+
+ for (i = 0; i < num_ios; i++) {
+ bdt_info = cmgr->io_bdt_pool[i];
+ bdt_info->bd_tbl = dma_alloc_coherent(&qedf->pdev->dev,
+ QEDF_MAX_BDS_PER_CMD * sizeof(struct fcoe_sge),
+ &bdt_info->bd_tbl_dma, GFP_KERNEL);
+ if (!bdt_info->bd_tbl) {
+ QEDF_WARN(&(qedf->dbg_ctx), "Failed to alloc "
+ "bdt_tbl[%d].\n", i);
+ goto mem_err;
+ }
+ }
+ atomic_set(&cmgr->free_list_cnt, num_ios);
+ QEDF_INFO(&(qedf->dbg_ctx), QEDF_LOG_IO,
+ "cmgr->free_list_cnt=%d.\n",
+ atomic_read(&cmgr->free_list_cnt));
+
+ return cmgr;
+
+mem_err:
+ qedf_cmd_mgr_free(cmgr);
+ return NULL;
+}
+
+struct qedf_ioreq *qedf_alloc_cmd(struct qedf_rport *fcport, u8 cmd_type)
+{
+ struct qedf_ctx *qedf = fcport->qedf;
+ struct qedf_cmd_mgr *cmd_mgr = qedf->cmd_mgr;
+ struct qedf_ioreq *io_req = NULL;
+ struct io_bdt *bd_tbl;
+ u16 xid;
+ uint32_t free_sqes;
+ int i;
+ unsigned long flags;
+
+ free_sqes = atomic_read(&fcport->free_sqes);
+
+ if (!free_sqes) {
+ QEDF_INFO(&(qedf->dbg_ctx), QEDF_LOG_IO,
+ "Returning NULL, free_sqes=%d.\n ",
+ free_sqes);
+ goto out_failed;
+ }
+
+ /* Limit the number of outstanding R/W tasks */
+ if ((atomic_read(&fcport->num_active_ios) >=
+ NUM_RW_TASKS_PER_CONNECTION)) {
+ QEDF_INFO(&(qedf->dbg_ctx), QEDF_LOG_IO,
+ "Returning NULL, num_active_ios=%d.\n",
+ atomic_read(&fcport->num_active_ios));
+ goto out_failed;
+ }
+
+ /* Limit global TIDs certain tasks */
+ if (atomic_read(&cmd_mgr->free_list_cnt) <= GBL_RSVD_TASKS) {
+ QEDF_INFO(&(qedf->dbg_ctx), QEDF_LOG_IO,
+ "Returning NULL, free_list_cnt=%d.\n",
+ atomic_read(&cmd_mgr->free_list_cnt));
+ goto out_failed;
+ }
+
+ spin_lock_irqsave(&cmd_mgr->lock, flags);
+ for (i = 0; i < FCOE_PARAMS_NUM_TASKS; i++) {
+ io_req = &cmd_mgr->cmds[cmd_mgr->idx];
+ cmd_mgr->idx++;
+ if (cmd_mgr->idx == FCOE_PARAMS_NUM_TASKS)
+ cmd_mgr->idx = 0;
+
+ /* Check to make sure command was previously freed */
+ if (!test_bit(QEDF_CMD_OUTSTANDING, &io_req->flags))
+ break;
+ }
+
+ if (i == FCOE_PARAMS_NUM_TASKS) {
+ spin_unlock_irqrestore(&cmd_mgr->lock, flags);
+ goto out_failed;
+ }
+
+ set_bit(QEDF_CMD_OUTSTANDING, &io_req->flags);
+ spin_unlock_irqrestore(&cmd_mgr->lock, flags);
+
+ atomic_inc(&fcport->num_active_ios);
+ atomic_dec(&fcport->free_sqes);
+ xid = io_req->xid;
+ atomic_dec(&cmd_mgr->free_list_cnt);
+
+ io_req->cmd_mgr = cmd_mgr;
+ io_req->fcport = fcport;
+
+ /* Hold the io_req against deletion */
+ kref_init(&io_req->refcount);
+
+ /* Bind io_bdt for this io_req */
+ /* Have a static link between io_req and io_bdt_pool */
+ bd_tbl = io_req->bd_tbl = cmd_mgr->io_bdt_pool[xid];
+ if (bd_tbl == NULL) {
+ QEDF_ERR(&(qedf->dbg_ctx), "bd_tbl is NULL, xid=%x.\n", xid);
+ kref_put(&io_req->refcount, qedf_release_cmd);
+ goto out_failed;
+ }
+ bd_tbl->io_req = io_req;
+ io_req->cmd_type = cmd_type;
+
+ /* Reset sequence offset data */
+ io_req->rx_buf_off = 0;
+ io_req->tx_buf_off = 0;
+ io_req->rx_id = 0xffff; /* No OX_ID */
+
+ return io_req;
+
+out_failed:
+ /* Record failure for stats and return NULL to caller */
+ qedf->alloc_failures++;
+ return NULL;
+}
+
+static void qedf_free_mp_resc(struct qedf_ioreq *io_req)
+{
+ struct qedf_mp_req *mp_req = &(io_req->mp_req);
+ struct qedf_ctx *qedf = io_req->fcport->qedf;
+ uint64_t sz = sizeof(struct fcoe_sge);
+
+ /* clear tm flags */
+ mp_req->tm_flags = 0;
+ if (mp_req->mp_req_bd) {
+ dma_free_coherent(&qedf->pdev->dev, sz,
+ mp_req->mp_req_bd, mp_req->mp_req_bd_dma);
+ mp_req->mp_req_bd = NULL;
+ }
+ if (mp_req->mp_resp_bd) {
+ dma_free_coherent(&qedf->pdev->dev, sz,
+ mp_req->mp_resp_bd, mp_req->mp_resp_bd_dma);
+ mp_req->mp_resp_bd = NULL;
+ }
+ if (mp_req->req_buf) {
+ dma_free_coherent(&qedf->pdev->dev, QEDF_PAGE_SIZE,
+ mp_req->req_buf, mp_req->req_buf_dma);
+ mp_req->req_buf = NULL;
+ }
+ if (mp_req->resp_buf) {
+ dma_free_coherent(&qedf->pdev->dev, QEDF_PAGE_SIZE,
+ mp_req->resp_buf, mp_req->resp_buf_dma);
+ mp_req->resp_buf = NULL;
+ }
+}
+
+void qedf_release_cmd(struct kref *ref)
+{
+ struct qedf_ioreq *io_req =
+ container_of(ref, struct qedf_ioreq, refcount);
+ struct qedf_cmd_mgr *cmd_mgr = io_req->cmd_mgr;
+ struct qedf_rport *fcport = io_req->fcport;
+
+ if (io_req->cmd_type == QEDF_ELS ||
+ io_req->cmd_type == QEDF_TASK_MGMT_CMD)
+ qedf_free_mp_resc(io_req);
+
+ atomic_inc(&cmd_mgr->free_list_cnt);
+ atomic_dec(&fcport->num_active_ios);
+ if (atomic_read(&fcport->num_active_ios) < 0)
+ QEDF_WARN(&(fcport->qedf->dbg_ctx), "active_ios < 0.\n");
+
+ /* Increment task retry identifier now that the request is released */
+ io_req->task_retry_identifier++;
+
+ clear_bit(QEDF_CMD_OUTSTANDING, &io_req->flags);
+}
+
+static int qedf_split_bd(struct qedf_ioreq *io_req, u64 addr, int sg_len,
+ int bd_index)
+{
+ struct fcoe_sge *bd = io_req->bd_tbl->bd_tbl;
+ int frag_size, sg_frags;
+
+ sg_frags = 0;
+ while (sg_len) {
+ if (sg_len > QEDF_BD_SPLIT_SZ)
+ frag_size = QEDF_BD_SPLIT_SZ;
+ else
+ frag_size = sg_len;
+ bd[bd_index + sg_frags].sge_addr.lo = U64_LO(addr);
+ bd[bd_index + sg_frags].sge_addr.hi = U64_HI(addr);
+ bd[bd_index + sg_frags].size = (uint16_t)frag_size;
+
+ addr += (u64)frag_size;
+ sg_frags++;
+ sg_len -= frag_size;
+ }
+ return sg_frags;
+}
+
+static int qedf_map_sg(struct qedf_ioreq *io_req)
+{
+ struct scsi_cmnd *sc = io_req->sc_cmd;
+ struct Scsi_Host *host = sc->device->host;
+ struct fc_lport *lport = shost_priv(host);
+ struct qedf_ctx *qedf = lport_priv(lport);
+ struct fcoe_sge *bd = io_req->bd_tbl->bd_tbl;
+ struct scatterlist *sg;
+ int byte_count = 0;
+ int sg_count = 0;
+ int bd_count = 0;
+ int sg_frags;
+ unsigned int sg_len;
+ u64 addr, end_addr;
+ int i;
+
+ sg_count = dma_map_sg(&qedf->pdev->dev, scsi_sglist(sc),
+ scsi_sg_count(sc), sc->sc_data_direction);
+
+ sg = scsi_sglist(sc);
+
+ /*
+ * New condition to send single SGE as cached-SGL with length less
+ * than 64k.
+ */
+ if ((sg_count == 1) && (sg_dma_len(sg) <=
+ QEDF_MAX_SGLEN_FOR_CACHESGL)) {
+ sg_len = sg_dma_len(sg);
+ addr = (u64)sg_dma_address(sg);
+
+ bd[bd_count].sge_addr.lo = (addr & 0xffffffff);
+ bd[bd_count].sge_addr.hi = (addr >> 32);
+ bd[bd_count].size = (u16)sg_len;
+
+ return ++bd_count;
+ }
+
+ scsi_for_each_sg(sc, sg, sg_count, i) {
+ sg_len = sg_dma_len(sg);
+ addr = (u64)sg_dma_address(sg);
+ end_addr = (u64)(addr + sg_len);
+
+ /*
+ * First s/g element in the list so check if the end_addr
+ * is paged aligned. Also check to make sure the length is
+ * at least page size.
+ */
+ if ((i == 0) && (sg_count > 1) &&
+ ((end_addr % QEDF_PAGE_SIZE) ||
+ sg_len < QEDF_PAGE_SIZE))
+ io_req->use_slowpath = true;
+ /*
+ * Last s/g element so check if the start address is paged
+ * aligned.
+ */
+ else if ((i == (sg_count - 1)) && (sg_count > 1) &&
+ (addr % QEDF_PAGE_SIZE))
+ io_req->use_slowpath = true;
+ /*
+ * Intermediate s/g element so check if start and end address
+ * is page aligned.
+ */
+ else if ((i != 0) && (i != (sg_count - 1)) &&
+ ((addr % QEDF_PAGE_SIZE) || (end_addr % QEDF_PAGE_SIZE)))
+ io_req->use_slowpath = true;
+
+ if (sg_len > QEDF_MAX_BD_LEN) {
+ sg_frags = qedf_split_bd(io_req, addr, sg_len,
+ bd_count);
+ } else {
+ sg_frags = 1;
+ bd[bd_count].sge_addr.lo = U64_LO(addr);
+ bd[bd_count].sge_addr.hi = U64_HI(addr);
+ bd[bd_count].size = (uint16_t)sg_len;
+ }
+
+ bd_count += sg_frags;
+ byte_count += sg_len;
+ }
+
+ if (byte_count != scsi_bufflen(sc))
+ QEDF_ERR(&(qedf->dbg_ctx), "byte_count = %d != "
+ "scsi_bufflen = %d, task_id = 0x%x.\n", byte_count,
+ scsi_bufflen(sc), io_req->xid);
+
+ return bd_count;
+}
+
+static int qedf_build_bd_list_from_sg(struct qedf_ioreq *io_req)
+{
+ struct scsi_cmnd *sc = io_req->sc_cmd;
+ struct fcoe_sge *bd = io_req->bd_tbl->bd_tbl;
+ int bd_count;
+
+ if (scsi_sg_count(sc)) {
+ bd_count = qedf_map_sg(io_req);
+ if (bd_count == 0)
+ return -ENOMEM;
+ } else {
+ bd_count = 0;
+ bd[0].sge_addr.lo = bd[0].sge_addr.hi = 0;
+ bd[0].size = 0;
+ }
+ io_req->bd_tbl->bd_valid = bd_count;
+
+ return 0;
+}
+
+static void qedf_build_fcp_cmnd(struct qedf_ioreq *io_req,
+ struct fcp_cmnd *fcp_cmnd)
+{
+ struct scsi_cmnd *sc_cmd = io_req->sc_cmd;
+
+ /* fcp_cmnd is 32 bytes */
+ memset(fcp_cmnd, 0, FCP_CMND_LEN);
+
+ /* 8 bytes: SCSI LUN info */
+ int_to_scsilun(sc_cmd->device->lun,
+ (struct scsi_lun *)&fcp_cmnd->fc_lun);
+
+ /* 4 bytes: flag info */
+ fcp_cmnd->fc_pri_ta = 0;
+ fcp_cmnd->fc_tm_flags = io_req->mp_req.tm_flags;
+ fcp_cmnd->fc_flags = io_req->io_req_flags;
+ fcp_cmnd->fc_cmdref = 0;
+
+ /* Populate data direction */
+ if (sc_cmd->sc_data_direction == DMA_TO_DEVICE)
+ fcp_cmnd->fc_flags |= FCP_CFL_WRDATA;
+ else if (sc_cmd->sc_data_direction == DMA_FROM_DEVICE)
+ fcp_cmnd->fc_flags |= FCP_CFL_RDDATA;
+
+ fcp_cmnd->fc_pri_ta = FCP_PTA_SIMPLE;
+
+ /* 16 bytes: CDB information */
+ memcpy(fcp_cmnd->fc_cdb, sc_cmd->cmnd, sc_cmd->cmd_len);
+
+ /* 4 bytes: FCP data length */
+ fcp_cmnd->fc_dl = htonl(io_req->data_xfer_len);
+
+}
+
+static void qedf_init_task(struct qedf_rport *fcport, struct fc_lport *lport,
+ struct qedf_ioreq *io_req, u32 *ptu_invalidate,
+ struct fcoe_task_context *task_ctx)
+{
+ enum fcoe_task_type task_type;
+ struct scsi_cmnd *sc_cmd = io_req->sc_cmd;
+ struct io_bdt *bd_tbl = io_req->bd_tbl;
+ union fcoe_data_desc_ctx *data_desc;
+ u32 *fcp_cmnd;
+ u32 tmp_fcp_cmnd[8];
+ int cnt, i;
+ int bd_count;
+ struct qedf_ctx *qedf = fcport->qedf;
+ uint16_t cq_idx = smp_processor_id() % qedf->num_queues;
+ u8 tmp_sgl_mode = 0;
+ u8 mst_sgl_mode = 0;
+
+ memset(task_ctx, 0, sizeof(struct fcoe_task_context));
+ io_req->task = task_ctx;
+
+ if (sc_cmd->sc_data_direction == DMA_TO_DEVICE)
+ task_type = FCOE_TASK_TYPE_WRITE_INITIATOR;
+ else
+ task_type = FCOE_TASK_TYPE_READ_INITIATOR;
+
+ /* Y Storm context */
+ task_ctx->ystorm_st_context.expect_first_xfer = 1;
+ task_ctx->ystorm_st_context.data_2_trns_rem = io_req->data_xfer_len;
+ /* Check if this is required */
+ task_ctx->ystorm_st_context.ox_id = io_req->xid;
+ task_ctx->ystorm_st_context.task_rety_identifier =
+ io_req->task_retry_identifier;
+
+ /* T Storm ag context */
+ SET_FIELD(task_ctx->tstorm_ag_context.flags0,
+ TSTORM_FCOE_TASK_AG_CTX_CONNECTION_TYPE, PROTOCOLID_FCOE);
+ task_ctx->tstorm_ag_context.icid = (u16)fcport->fw_cid;
+
+ /* T Storm st context */
+ SET_FIELD(task_ctx->tstorm_st_context.read_write.flags,
+ FCOE_TSTORM_FCOE_TASK_ST_CTX_READ_WRITE_EXP_FIRST_FRAME,
+ 1);
+ task_ctx->tstorm_st_context.read_write.rx_id = 0xffff;
+
+ task_ctx->tstorm_st_context.read_only.dev_type =
+ FCOE_TASK_DEV_TYPE_DISK;
+ task_ctx->tstorm_st_context.read_only.conf_supported = 0;
+ task_ctx->tstorm_st_context.read_only.cid = fcport->fw_cid;
+
+ /* Completion queue for response. */
+ task_ctx->tstorm_st_context.read_only.glbl_q_num = cq_idx;
+ task_ctx->tstorm_st_context.read_only.fcp_cmd_trns_size =
+ io_req->data_xfer_len;
+ task_ctx->tstorm_st_context.read_write.e_d_tov_exp_timeout_val =
+ lport->e_d_tov;
+
+ task_ctx->ustorm_ag_context.global_cq_num = cq_idx;
+ io_req->fp_idx = cq_idx;
+
+ bd_count = bd_tbl->bd_valid;
+ if (task_type == FCOE_TASK_TYPE_WRITE_INITIATOR) {
+ /* Setup WRITE task */
+ struct fcoe_sge *fcoe_bd_tbl = bd_tbl->bd_tbl;
+
+ task_ctx->ystorm_st_context.task_type =
+ FCOE_TASK_TYPE_WRITE_INITIATOR;
+ data_desc = &task_ctx->ystorm_st_context.data_desc;
+
+ if (io_req->use_slowpath) {
+ SET_FIELD(task_ctx->ystorm_st_context.sgl_mode,
+ YSTORM_FCOE_TASK_ST_CTX_TX_SGL_MODE,
+ FCOE_SLOW_SGL);
+ data_desc->slow.base_sgl_addr.lo =
+ U64_LO(bd_tbl->bd_tbl_dma);
+ data_desc->slow.base_sgl_addr.hi =
+ U64_HI(bd_tbl->bd_tbl_dma);
+ data_desc->slow.remainder_num_sges = bd_count;
+ data_desc->slow.curr_sge_off = 0;
+ data_desc->slow.curr_sgl_index = 0;
+ qedf->slow_sge_ios++;
+ io_req->sge_type = QEDF_IOREQ_SLOW_SGE;
+ } else {
+ SET_FIELD(task_ctx->ystorm_st_context.sgl_mode,
+ YSTORM_FCOE_TASK_ST_CTX_TX_SGL_MODE,
+ (bd_count <= 4) ? (enum fcoe_sgl_mode)bd_count :
+ FCOE_MUL_FAST_SGES);
+
+ if (bd_count == 1) {
+ data_desc->single_sge.sge_addr.lo =
+ fcoe_bd_tbl->sge_addr.lo;
+ data_desc->single_sge.sge_addr.hi =
+ fcoe_bd_tbl->sge_addr.hi;
+ data_desc->single_sge.size =
+ fcoe_bd_tbl->size;
+ data_desc->single_sge.is_valid_sge = 0;
+ qedf->single_sge_ios++;
+ io_req->sge_type = QEDF_IOREQ_SINGLE_SGE;
+ } else {
+ data_desc->fast.sgl_start_addr.lo =
+ U64_LO(bd_tbl->bd_tbl_dma);
+ data_desc->fast.sgl_start_addr.hi =
+ U64_HI(bd_tbl->bd_tbl_dma);
+ data_desc->fast.sgl_byte_offset =
+ data_desc->fast.sgl_start_addr.lo &
+ (QEDF_PAGE_SIZE - 1);
+ if (data_desc->fast.sgl_byte_offset > 0)
+ QEDF_ERR(&(qedf->dbg_ctx),
+ "byte_offset=%u for xid=0x%x.\n",
+ io_req->xid,
+ data_desc->fast.sgl_byte_offset);
+ data_desc->fast.task_reuse_cnt =
+ io_req->reuse_count;
+ io_req->reuse_count++;
+ if (io_req->reuse_count == QEDF_MAX_REUSE) {
+ *ptu_invalidate = 1;
+ io_req->reuse_count = 0;
+ }
+ qedf->fast_sge_ios++;
+ io_req->sge_type = QEDF_IOREQ_FAST_SGE;
+ }
+ }
+
+ /* T Storm context */
+ task_ctx->tstorm_st_context.read_only.task_type =
+ FCOE_TASK_TYPE_WRITE_INITIATOR;
+
+ /* M Storm context */
+ tmp_sgl_mode = GET_FIELD(task_ctx->ystorm_st_context.sgl_mode,
+ YSTORM_FCOE_TASK_ST_CTX_TX_SGL_MODE);
+ SET_FIELD(task_ctx->mstorm_st_context.non_fp.tx_rx_sgl_mode,
+ FCOE_MSTORM_FCOE_TASK_ST_CTX_NON_FP_TX_SGL_MODE,
+ tmp_sgl_mode);
+
+ } else {
+ /* Setup READ task */
+
+ /* M Storm context */
+ struct fcoe_sge *fcoe_bd_tbl = bd_tbl->bd_tbl;
+
+ data_desc = &task_ctx->mstorm_st_context.fp.data_desc;
+ task_ctx->mstorm_st_context.fp.data_2_trns_rem =
+ io_req->data_xfer_len;
+
+ if (io_req->use_slowpath) {
+ SET_FIELD(
+ task_ctx->mstorm_st_context.non_fp.tx_rx_sgl_mode,
+ FCOE_MSTORM_FCOE_TASK_ST_CTX_NON_FP_RX_SGL_MODE,
+ FCOE_SLOW_SGL);
+ data_desc->slow.base_sgl_addr.lo =
+ U64_LO(bd_tbl->bd_tbl_dma);
+ data_desc->slow.base_sgl_addr.hi =
+ U64_HI(bd_tbl->bd_tbl_dma);
+ data_desc->slow.remainder_num_sges =
+ bd_count;
+ data_desc->slow.curr_sge_off = 0;
+ data_desc->slow.curr_sgl_index = 0;
+ qedf->slow_sge_ios++;
+ io_req->sge_type = QEDF_IOREQ_SLOW_SGE;
+ } else {
+ SET_FIELD(
+ task_ctx->mstorm_st_context.non_fp.tx_rx_sgl_mode,
+ FCOE_MSTORM_FCOE_TASK_ST_CTX_NON_FP_RX_SGL_MODE,
+ (bd_count <= 4) ? (enum fcoe_sgl_mode)bd_count :
+ FCOE_MUL_FAST_SGES);
+
+ if (bd_count == 1) {
+ data_desc->single_sge.sge_addr.lo =
+ fcoe_bd_tbl->sge_addr.lo;
+ data_desc->single_sge.sge_addr.hi =
+ fcoe_bd_tbl->sge_addr.hi;
+ data_desc->single_sge.size =
+ fcoe_bd_tbl->size;
+ data_desc->single_sge.is_valid_sge = 0;
+ qedf->single_sge_ios++;
+ io_req->sge_type = QEDF_IOREQ_SINGLE_SGE;
+ } else {
+ data_desc->fast.sgl_start_addr.lo =
+ U64_LO(bd_tbl->bd_tbl_dma);
+ data_desc->fast.sgl_start_addr.hi =
+ U64_HI(bd_tbl->bd_tbl_dma);
+ data_desc->fast.sgl_byte_offset = 0;
+ data_desc->fast.task_reuse_cnt =
+ io_req->reuse_count;
+ io_req->reuse_count++;
+ if (io_req->reuse_count == QEDF_MAX_REUSE) {
+ *ptu_invalidate = 1;
+ io_req->reuse_count = 0;
+ }
+ qedf->fast_sge_ios++;
+ io_req->sge_type = QEDF_IOREQ_FAST_SGE;
+ }
+ }
+
+ /* Y Storm context */
+ task_ctx->ystorm_st_context.expect_first_xfer = 0;
+ task_ctx->ystorm_st_context.task_type =
+ FCOE_TASK_TYPE_READ_INITIATOR;
+
+ /* T Storm context */
+ task_ctx->tstorm_st_context.read_only.task_type =
+ FCOE_TASK_TYPE_READ_INITIATOR;
+ mst_sgl_mode = GET_FIELD(
+ task_ctx->mstorm_st_context.non_fp.tx_rx_sgl_mode,
+ FCOE_MSTORM_FCOE_TASK_ST_CTX_NON_FP_RX_SGL_MODE);
+ SET_FIELD(task_ctx->tstorm_st_context.read_write.flags,
+ FCOE_TSTORM_FCOE_TASK_ST_CTX_READ_WRITE_RX_SGL_MODE,
+ mst_sgl_mode);
+ }
+
+ /* fill FCP_CMND IU */
+ fcp_cmnd = (u32 *)task_ctx->ystorm_st_context.tx_info_union.fcp_cmd_payload.opaque;
+ qedf_build_fcp_cmnd(io_req, (struct fcp_cmnd *)&tmp_fcp_cmnd);
+
+ /* Swap fcp_cmnd since FC is big endian */
+ cnt = sizeof(struct fcp_cmnd) / sizeof(u32);
+
+ for (i = 0; i < cnt; i++) {
+ *fcp_cmnd = cpu_to_be32(tmp_fcp_cmnd[i]);
+ fcp_cmnd++;
+ }
+
+ /* M Storm context - Sense buffer */
+ task_ctx->mstorm_st_context.non_fp.rsp_buf_addr.lo =
+ U64_LO(io_req->sense_buffer_dma);
+ task_ctx->mstorm_st_context.non_fp.rsp_buf_addr.hi =
+ U64_HI(io_req->sense_buffer_dma);
+}
+
+void qedf_init_mp_task(struct qedf_ioreq *io_req,
+ struct fcoe_task_context *task_ctx)
+{
+ struct qedf_mp_req *mp_req = &(io_req->mp_req);
+ struct qedf_rport *fcport = io_req->fcport;
+ struct qedf_ctx *qedf = io_req->fcport->qedf;
+ struct fc_frame_header *fc_hdr;
+ enum fcoe_task_type task_type = 0;
+ union fcoe_data_desc_ctx *data_desc;
+
+ QEDF_INFO(&(qedf->dbg_ctx), QEDF_LOG_DISC, "Initializing MP task "
+ "for cmd_type = %d\n", io_req->cmd_type);
+
+ qedf->control_requests++;
+
+ /* Obtain task_type */
+ if ((io_req->cmd_type == QEDF_TASK_MGMT_CMD) ||
+ (io_req->cmd_type == QEDF_ELS)) {
+ task_type = FCOE_TASK_TYPE_MIDPATH;
+ } else if (io_req->cmd_type == QEDF_ABTS) {
+ task_type = FCOE_TASK_TYPE_ABTS;
+ }
+
+ memset(task_ctx, 0, sizeof(struct fcoe_task_context));
+
+ /* Setup the task from io_req for easy reference */
+ io_req->task = task_ctx;
+
+ QEDF_INFO(&(qedf->dbg_ctx), QEDF_LOG_DISC, "task type = %d\n",
+ task_type);
+
+ /* YSTORM only */
+ {
+ /* Initialize YSTORM task context */
+ struct fcoe_tx_mid_path_params *task_fc_hdr =
+ &task_ctx->ystorm_st_context.tx_info_union.tx_params.mid_path;
+ memset(task_fc_hdr, 0, sizeof(struct fcoe_tx_mid_path_params));
+ task_ctx->ystorm_st_context.task_rety_identifier =
+ io_req->task_retry_identifier;
+
+ /* Init SGL parameters */
+ if ((task_type == FCOE_TASK_TYPE_MIDPATH) ||
+ (task_type == FCOE_TASK_TYPE_UNSOLICITED)) {
+ data_desc = &task_ctx->ystorm_st_context.data_desc;
+ data_desc->slow.base_sgl_addr.lo =
+ U64_LO(mp_req->mp_req_bd_dma);
+ data_desc->slow.base_sgl_addr.hi =
+ U64_HI(mp_req->mp_req_bd_dma);
+ data_desc->slow.remainder_num_sges = 1;
+ data_desc->slow.curr_sge_off = 0;
+ data_desc->slow.curr_sgl_index = 0;
+ }
+
+ fc_hdr = &(mp_req->req_fc_hdr);
+ if (task_type == FCOE_TASK_TYPE_MIDPATH) {
+ fc_hdr->fh_ox_id = io_req->xid;
+ fc_hdr->fh_rx_id = htons(0xffff);
+ } else if (task_type == FCOE_TASK_TYPE_UNSOLICITED) {
+ fc_hdr->fh_rx_id = io_req->xid;
+ }
+
+ /* Fill FC Header into middle path buffer */
+ task_fc_hdr->parameter = fc_hdr->fh_parm_offset;
+ task_fc_hdr->r_ctl = fc_hdr->fh_r_ctl;
+ task_fc_hdr->type = fc_hdr->fh_type;
+ task_fc_hdr->cs_ctl = fc_hdr->fh_cs_ctl;
+ task_fc_hdr->df_ctl = fc_hdr->fh_df_ctl;
+ task_fc_hdr->rx_id = fc_hdr->fh_rx_id;
+ task_fc_hdr->ox_id = fc_hdr->fh_ox_id;
+
+ task_ctx->ystorm_st_context.data_2_trns_rem =
+ io_req->data_xfer_len;
+ task_ctx->ystorm_st_context.task_type = task_type;
+ }
+
+ /* TSTORM ONLY */
+ {
+ task_ctx->tstorm_ag_context.icid = (u16)fcport->fw_cid;
+ task_ctx->tstorm_st_context.read_only.cid = fcport->fw_cid;
+ /* Always send middle-path repsonses on CQ #0 */
+ task_ctx->tstorm_st_context.read_only.glbl_q_num = 0;
+ io_req->fp_idx = 0;
+ SET_FIELD(task_ctx->tstorm_ag_context.flags0,
+ TSTORM_FCOE_TASK_AG_CTX_CONNECTION_TYPE,
+ PROTOCOLID_FCOE);
+ task_ctx->tstorm_st_context.read_only.task_type = task_type;
+ SET_FIELD(task_ctx->tstorm_st_context.read_write.flags,
+ FCOE_TSTORM_FCOE_TASK_ST_CTX_READ_WRITE_EXP_FIRST_FRAME,
+ 1);
+ task_ctx->tstorm_st_context.read_write.rx_id = 0xffff;
+ }
+
+ /* MSTORM only */
+ {
+ if (task_type == FCOE_TASK_TYPE_MIDPATH) {
+ /* Initialize task context */
+ data_desc = &task_ctx->mstorm_st_context.fp.data_desc;
+
+ /* Set cache sges address and length */
+ data_desc->slow.base_sgl_addr.lo =
+ U64_LO(mp_req->mp_resp_bd_dma);
+ data_desc->slow.base_sgl_addr.hi =
+ U64_HI(mp_req->mp_resp_bd_dma);
+ data_desc->slow.remainder_num_sges = 1;
+ data_desc->slow.curr_sge_off = 0;
+ data_desc->slow.curr_sgl_index = 0;
+
+ /*
+ * Also need to fil in non-fastpath response address
+ * for middle path commands.
+ */
+ task_ctx->mstorm_st_context.non_fp.rsp_buf_addr.lo =
+ U64_LO(mp_req->mp_resp_bd_dma);
+ task_ctx->mstorm_st_context.non_fp.rsp_buf_addr.hi =
+ U64_HI(mp_req->mp_resp_bd_dma);
+ }
+ }
+
+ /* USTORM ONLY */
+ {
+ task_ctx->ustorm_ag_context.global_cq_num = 0;
+ }
+
+ /* I/O stats. Middle path commands always use slow SGEs */
+ qedf->slow_sge_ios++;
+ io_req->sge_type = QEDF_IOREQ_SLOW_SGE;
+}
+
+void qedf_add_to_sq(struct qedf_rport *fcport, u16 xid, u32 ptu_invalidate,
+ enum fcoe_task_type req_type, u32 offset)
+{
+ struct fcoe_wqe *sqe;
+ uint16_t total_sqe = (fcport->sq_mem_size)/(sizeof(struct fcoe_wqe));
+
+ sqe = &fcport->sq[fcport->sq_prod_idx];
+
+ fcport->sq_prod_idx++;
+ fcport->fw_sq_prod_idx++;
+ if (fcport->sq_prod_idx == total_sqe)
+ fcport->sq_prod_idx = 0;
+
+ switch (req_type) {
+ case FCOE_TASK_TYPE_WRITE_INITIATOR:
+ case FCOE_TASK_TYPE_READ_INITIATOR:
+ SET_FIELD(sqe->flags, FCOE_WQE_REQ_TYPE, SEND_FCOE_CMD);
+ if (ptu_invalidate)
+ SET_FIELD(sqe->flags, FCOE_WQE_INVALIDATE_PTU, 1);
+ break;
+ case FCOE_TASK_TYPE_MIDPATH:
+ SET_FIELD(sqe->flags, FCOE_WQE_REQ_TYPE, SEND_FCOE_MIDPATH);
+ break;
+ case FCOE_TASK_TYPE_ABTS:
+ SET_FIELD(sqe->flags, FCOE_WQE_REQ_TYPE,
+ SEND_FCOE_ABTS_REQUEST);
+ break;
+ case FCOE_TASK_TYPE_EXCHANGE_CLEANUP:
+ SET_FIELD(sqe->flags, FCOE_WQE_REQ_TYPE,
+ FCOE_EXCHANGE_CLEANUP);
+ break;
+ case FCOE_TASK_TYPE_SEQUENCE_CLEANUP:
+ SET_FIELD(sqe->flags, FCOE_WQE_REQ_TYPE,
+ FCOE_SEQUENCE_RECOVERY);
+ /* NOTE: offset param only used for sequence recovery */
+ sqe->additional_info_union.seq_rec_updated_offset = offset;
+ break;
+ case FCOE_TASK_TYPE_UNSOLICITED:
+ break;
+ default:
+ break;
+ }
+
+ sqe->task_id = xid;
+
+ /* Make sure SQ data is coherent */
+ wmb();
+
+}
+
+void qedf_ring_doorbell(struct qedf_rport *fcport)
+{
+ struct fcoe_db_data dbell = { 0 };
+
+ dbell.agg_flags = 0;
+
+ dbell.params |= DB_DEST_XCM << FCOE_DB_DATA_DEST_SHIFT;
+ dbell.params |= DB_AGG_CMD_SET << FCOE_DB_DATA_AGG_CMD_SHIFT;
+ dbell.params |= DQ_XCM_FCOE_SQ_PROD_CMD <<
+ FCOE_DB_DATA_AGG_VAL_SEL_SHIFT;
+
+ dbell.sq_prod = fcport->fw_sq_prod_idx;
+ writel(*(u32 *)&dbell, fcport->p_doorbell);
+ /* Make sure SQ index is updated so f/w prcesses requests in order */
+ wmb();
+ mmiowb();
+}
+
+static void qedf_trace_io(struct qedf_rport *fcport, struct qedf_ioreq *io_req,
+ int8_t direction)
+{
+ struct qedf_ctx *qedf = fcport->qedf;
+ struct qedf_io_log *io_log;
+ struct scsi_cmnd *sc_cmd = io_req->sc_cmd;
+ unsigned long flags;
+ uint8_t op;
+
+ spin_lock_irqsave(&qedf->io_trace_lock, flags);
+
+ io_log = &qedf->io_trace_buf[qedf->io_trace_idx];
+ io_log->direction = direction;
+ io_log->task_id = io_req->xid;
+ io_log->port_id = fcport->rdata->ids.port_id;
+ io_log->lun = sc_cmd->device->lun;
+ io_log->op = op = sc_cmd->cmnd[0];
+ io_log->lba[0] = sc_cmd->cmnd[2];
+ io_log->lba[1] = sc_cmd->cmnd[3];
+ io_log->lba[2] = sc_cmd->cmnd[4];
+ io_log->lba[3] = sc_cmd->cmnd[5];
+ io_log->bufflen = scsi_bufflen(sc_cmd);
+ io_log->sg_count = scsi_sg_count(sc_cmd);
+ io_log->result = sc_cmd->result;
+ io_log->jiffies = jiffies;
+ io_log->refcount = kref_read(&io_req->refcount);
+
+ if (direction == QEDF_IO_TRACE_REQ) {
+ /* For requests we only care abot the submission CPU */
+ io_log->req_cpu = io_req->cpu;
+ io_log->int_cpu = 0;
+ io_log->rsp_cpu = 0;
+ } else if (direction == QEDF_IO_TRACE_RSP) {
+ io_log->req_cpu = io_req->cpu;
+ io_log->int_cpu = io_req->int_cpu;
+ io_log->rsp_cpu = smp_processor_id();
+ }
+
+ io_log->sge_type = io_req->sge_type;
+
+ qedf->io_trace_idx++;
+ if (qedf->io_trace_idx == QEDF_IO_TRACE_SIZE)
+ qedf->io_trace_idx = 0;
+
+ spin_unlock_irqrestore(&qedf->io_trace_lock, flags);
+}
+
+int qedf_post_io_req(struct qedf_rport *fcport, struct qedf_ioreq *io_req)
+{
+ struct scsi_cmnd *sc_cmd = io_req->sc_cmd;
+ struct Scsi_Host *host = sc_cmd->device->host;
+ struct fc_lport *lport = shost_priv(host);
+ struct qedf_ctx *qedf = lport_priv(lport);
+ struct fcoe_task_context *task_ctx;
+ u16 xid;
+ enum fcoe_task_type req_type = 0;
+ u32 ptu_invalidate = 0;
+
+ /* Initialize rest of io_req fileds */
+ io_req->data_xfer_len = scsi_bufflen(sc_cmd);
+ sc_cmd->SCp.ptr = (char *)io_req;
+ io_req->use_slowpath = false; /* Assume fast SGL by default */
+
+ /* Record which cpu this request is associated with */
+ io_req->cpu = smp_processor_id();
+
+ if (sc_cmd->sc_data_direction == DMA_FROM_DEVICE) {
+ req_type = FCOE_TASK_TYPE_READ_INITIATOR;
+ io_req->io_req_flags = QEDF_READ;
+ qedf->input_requests++;
+ } else if (sc_cmd->sc_data_direction == DMA_TO_DEVICE) {
+ req_type = FCOE_TASK_TYPE_WRITE_INITIATOR;
+ io_req->io_req_flags = QEDF_WRITE;
+ qedf->output_requests++;
+ } else {
+ io_req->io_req_flags = 0;
+ qedf->control_requests++;
+ }
+
+ xid = io_req->xid;
+
+ /* Build buffer descriptor list for firmware from sg list */
+ if (qedf_build_bd_list_from_sg(io_req)) {
+ QEDF_ERR(&(qedf->dbg_ctx), "BD list creation failed.\n");
+ kref_put(&io_req->refcount, qedf_release_cmd);
+ return -EAGAIN;
+ }
+
+ /* Get the task context */
+ task_ctx = qedf_get_task_mem(&qedf->tasks, xid);
+ if (!task_ctx) {
+ QEDF_WARN(&(qedf->dbg_ctx), "task_ctx is NULL, xid=%d.\n",
+ xid);
+ kref_put(&io_req->refcount, qedf_release_cmd);
+ return -EINVAL;
+ }
+
+ qedf_init_task(fcport, lport, io_req, &ptu_invalidate, task_ctx);
+
+ if (!test_bit(QEDF_RPORT_SESSION_READY, &fcport->flags)) {
+ QEDF_ERR(&(qedf->dbg_ctx), "Session not offloaded yet.\n");
+ kref_put(&io_req->refcount, qedf_release_cmd);
+ }
+
+ /* Obtain free SQ entry */
+ qedf_add_to_sq(fcport, xid, ptu_invalidate, req_type, 0);
+
+ /* Ring doorbell */
+ qedf_ring_doorbell(fcport);
+
+ if (qedf_io_tracing && io_req->sc_cmd)
+ qedf_trace_io(fcport, io_req, QEDF_IO_TRACE_REQ);
+
+ return false;
+}
+
+int
+qedf_queuecommand(struct Scsi_Host *host, struct scsi_cmnd *sc_cmd)
+{
+ struct fc_lport *lport = shost_priv(host);
+ struct qedf_ctx *qedf = lport_priv(lport);
+ struct fc_rport *rport = starget_to_rport(scsi_target(sc_cmd->device));
+ struct fc_rport_libfc_priv *rp = rport->dd_data;
+ struct qedf_rport *fcport = rport->dd_data;
+ struct qedf_ioreq *io_req;
+ int rc = 0;
+ int rval;
+ unsigned long flags = 0;
+
+
+ if (test_bit(QEDF_UNLOADING, &qedf->flags) ||
+ test_bit(QEDF_DBG_STOP_IO, &qedf->flags)) {
+ sc_cmd->result = DID_NO_CONNECT << 16;
+ sc_cmd->scsi_done(sc_cmd);
+ return 0;
+ }
+
+ rval = fc_remote_port_chkready(rport);
+ if (rval) {
+ sc_cmd->result = rval;
+ sc_cmd->scsi_done(sc_cmd);
+ return 0;
+ }
+
+ /* Retry command if we are doing a qed drain operation */
+ if (test_bit(QEDF_DRAIN_ACTIVE, &qedf->flags)) {
+ rc = SCSI_MLQUEUE_HOST_BUSY;
+ goto exit_qcmd;
+ }
+
+ if (lport->state != LPORT_ST_READY ||
+ atomic_read(&qedf->link_state) != QEDF_LINK_UP) {
+ rc = SCSI_MLQUEUE_HOST_BUSY;
+ goto exit_qcmd;
+ }
+
+ /* rport and tgt are allocated together, so tgt should be non-NULL */
+ fcport = (struct qedf_rport *)&rp[1];
+
+ if (!test_bit(QEDF_RPORT_SESSION_READY, &fcport->flags)) {
+ /*
+ * Session is not offloaded yet. Let SCSI-ml retry
+ * the command.
+ */
+ rc = SCSI_MLQUEUE_TARGET_BUSY;
+ goto exit_qcmd;
+ }
+ if (fcport->retry_delay_timestamp) {
+ if (time_after(jiffies, fcport->retry_delay_timestamp)) {
+ fcport->retry_delay_timestamp = 0;
+ } else {
+ /* If retry_delay timer is active, flow off the ML */
+ rc = SCSI_MLQUEUE_TARGET_BUSY;
+ goto exit_qcmd;
+ }
+ }
+
+ io_req = qedf_alloc_cmd(fcport, QEDF_SCSI_CMD);
+ if (!io_req) {
+ rc = SCSI_MLQUEUE_HOST_BUSY;
+ goto exit_qcmd;
+ }
+
+ io_req->sc_cmd = sc_cmd;
+
+ /* Take fcport->rport_lock for posting to fcport send queue */
+ spin_lock_irqsave(&fcport->rport_lock, flags);
+ if (qedf_post_io_req(fcport, io_req)) {
+ QEDF_WARN(&(qedf->dbg_ctx), "Unable to post io_req\n");
+ /* Return SQE to pool */
+ atomic_inc(&fcport->free_sqes);
+ rc = SCSI_MLQUEUE_HOST_BUSY;
+ }
+ spin_unlock_irqrestore(&fcport->rport_lock, flags);
+
+exit_qcmd:
+ return rc;
+}
+
+static void qedf_parse_fcp_rsp(struct qedf_ioreq *io_req,
+ struct fcoe_cqe_rsp_info *fcp_rsp)
+{
+ struct scsi_cmnd *sc_cmd = io_req->sc_cmd;
+ struct qedf_ctx *qedf = io_req->fcport->qedf;
+ u8 rsp_flags = fcp_rsp->rsp_flags.flags;
+ int fcp_sns_len = 0;
+ int fcp_rsp_len = 0;
+ uint8_t *rsp_info, *sense_data;
+
+ io_req->fcp_status = FC_GOOD;
+ io_req->fcp_resid = 0;
+ if (rsp_flags & (FCOE_FCP_RSP_FLAGS_FCP_RESID_OVER |
+ FCOE_FCP_RSP_FLAGS_FCP_RESID_UNDER))
+ io_req->fcp_resid = fcp_rsp->fcp_resid;
+
+ io_req->scsi_comp_flags = rsp_flags;
+ CMD_SCSI_STATUS(sc_cmd) = io_req->cdb_status =
+ fcp_rsp->scsi_status_code;
+
+ if (rsp_flags &
+ FCOE_FCP_RSP_FLAGS_FCP_RSP_LEN_VALID)
+ fcp_rsp_len = fcp_rsp->fcp_rsp_len;
+
+ if (rsp_flags &
+ FCOE_FCP_RSP_FLAGS_FCP_SNS_LEN_VALID)
+ fcp_sns_len = fcp_rsp->fcp_sns_len;
+
+ io_req->fcp_rsp_len = fcp_rsp_len;
+ io_req->fcp_sns_len = fcp_sns_len;
+ rsp_info = sense_data = io_req->sense_buffer;
+
+ /* fetch fcp_rsp_code */
+ if ((fcp_rsp_len == 4) || (fcp_rsp_len == 8)) {
+ /* Only for task management function */
+ io_req->fcp_rsp_code = rsp_info[3];
+ QEDF_INFO(&(qedf->dbg_ctx), QEDF_LOG_IO,
+ "fcp_rsp_code = %d\n", io_req->fcp_rsp_code);
+ /* Adjust sense-data location. */
+ sense_data += fcp_rsp_len;
+ }
+
+ if (fcp_sns_len > SCSI_SENSE_BUFFERSIZE) {
+ QEDF_INFO(&(qedf->dbg_ctx), QEDF_LOG_IO,
+ "Truncating sense buffer\n");
+ fcp_sns_len = SCSI_SENSE_BUFFERSIZE;
+ }
+
+ memset(sc_cmd->sense_buffer, 0, SCSI_SENSE_BUFFERSIZE);
+ if (fcp_sns_len)
+ memcpy(sc_cmd->sense_buffer, sense_data,
+ fcp_sns_len);
+}
+
+static void qedf_unmap_sg_list(struct qedf_ctx *qedf, struct qedf_ioreq *io_req)
+{
+ struct scsi_cmnd *sc = io_req->sc_cmd;
+
+ if (io_req->bd_tbl->bd_valid && sc && scsi_sg_count(sc)) {
+ dma_unmap_sg(&qedf->pdev->dev, scsi_sglist(sc),
+ scsi_sg_count(sc), sc->sc_data_direction);
+ io_req->bd_tbl->bd_valid = 0;
+ }
+}
+
+void qedf_scsi_completion(struct qedf_ctx *qedf, struct fcoe_cqe *cqe,
+ struct qedf_ioreq *io_req)
+{
+ u16 xid, rval;
+ struct fcoe_task_context *task_ctx;
+ struct scsi_cmnd *sc_cmd;
+ struct fcoe_cqe_rsp_info *fcp_rsp;
+ struct qedf_rport *fcport;
+ int refcount;
+ u16 scope, qualifier = 0;
+ u8 fw_residual_flag = 0;
+
+ if (!io_req)
+ return;
+ if (!cqe)
+ return;
+
+ xid = io_req->xid;
+ task_ctx = qedf_get_task_mem(&qedf->tasks, xid);
+ sc_cmd = io_req->sc_cmd;
+ fcp_rsp = &cqe->cqe_info.rsp_info;
+
+ if (!sc_cmd) {
+ QEDF_WARN(&(qedf->dbg_ctx), "sc_cmd is NULL!\n");
+ return;
+ }
+
+ if (!sc_cmd->SCp.ptr) {
+ QEDF_WARN(&(qedf->dbg_ctx), "SCp.ptr is NULL, returned in "
+ "another context.\n");
+ return;
+ }
+
+ if (!sc_cmd->request) {
+ QEDF_WARN(&(qedf->dbg_ctx), "sc_cmd->request is NULL, "
+ "sc_cmd=%p.\n", sc_cmd);
+ return;
+ }
+
+ if (!sc_cmd->request->special) {
+ QEDF_WARN(&(qedf->dbg_ctx), "request->special is NULL so "
+ "request not valid, sc_cmd=%p.\n", sc_cmd);
+ return;
+ }
+
+ if (!sc_cmd->request->q) {
+ QEDF_WARN(&(qedf->dbg_ctx), "request->q is NULL so request "
+ "is not valid, sc_cmd=%p.\n", sc_cmd);
+ return;
+ }
+
+ fcport = io_req->fcport;
+
+ qedf_parse_fcp_rsp(io_req, fcp_rsp);
+
+ qedf_unmap_sg_list(qedf, io_req);
+
+ /* Check for FCP transport error */
+ if (io_req->fcp_rsp_len > 3 && io_req->fcp_rsp_code) {
+ QEDF_ERR(&(qedf->dbg_ctx),
+ "FCP I/O protocol failure xid=0x%x fcp_rsp_len=%d "
+ "fcp_rsp_code=%d.\n", io_req->xid, io_req->fcp_rsp_len,
+ io_req->fcp_rsp_code);
+ sc_cmd->result = DID_BUS_BUSY << 16;
+ goto out;
+ }
+
+ fw_residual_flag = GET_FIELD(cqe->cqe_info.rsp_info.fw_error_flags,
+ FCOE_CQE_RSP_INFO_FW_UNDERRUN);
+ if (fw_residual_flag) {
+ QEDF_ERR(&(qedf->dbg_ctx),
+ "Firmware detected underrun: xid=0x%x fcp_rsp.flags=0x%02x "
+ "fcp_resid=%d fw_residual=0x%x.\n", io_req->xid,
+ fcp_rsp->rsp_flags.flags, io_req->fcp_resid,
+ cqe->cqe_info.rsp_info.fw_residual);
+
+ if (io_req->cdb_status == 0)
+ sc_cmd->result = (DID_ERROR << 16) | io_req->cdb_status;
+ else
+ sc_cmd->result = (DID_OK << 16) | io_req->cdb_status;
+
+ /* Abort the command since we did not get all the data */
+ init_completion(&io_req->abts_done);
+ rval = qedf_initiate_abts(io_req, true);
+ if (rval) {
+ QEDF_ERR(&(qedf->dbg_ctx), "Failed to queue ABTS.\n");
+ sc_cmd->result = (DID_ERROR << 16) | io_req->cdb_status;
+ }
+
+ /*
+ * Set resid to the whole buffer length so we won't try to resue
+ * any previously data.
+ */
+ scsi_set_resid(sc_cmd, scsi_bufflen(sc_cmd));
+ goto out;
+ }
+
+ switch (io_req->fcp_status) {
+ case FC_GOOD:
+ if (io_req->cdb_status == 0) {
+ /* Good I/O completion */
+ sc_cmd->result = DID_OK << 16;
+ } else {
+ refcount = kref_read(&io_req->refcount);
+ QEDF_INFO(&(qedf->dbg_ctx), QEDF_LOG_IO,
+ "%d:0:%d:%d xid=0x%0x op=0x%02x "
+ "lba=%02x%02x%02x%02x cdb_status=%d "
+ "fcp_resid=0x%x refcount=%d.\n",
+ qedf->lport->host->host_no, sc_cmd->device->id,
+ sc_cmd->device->lun, io_req->xid,
+ sc_cmd->cmnd[0], sc_cmd->cmnd[2], sc_cmd->cmnd[3],
+ sc_cmd->cmnd[4], sc_cmd->cmnd[5],
+ io_req->cdb_status, io_req->fcp_resid,
+ refcount);
+ sc_cmd->result = (DID_OK << 16) | io_req->cdb_status;
+
+ if (io_req->cdb_status == SAM_STAT_TASK_SET_FULL ||
+ io_req->cdb_status == SAM_STAT_BUSY) {
+ /*
+ * Check whether we need to set retry_delay at
+ * all based on retry_delay module parameter
+ * and the status qualifier.
+ */
+
+ /* Upper 2 bits */
+ scope = fcp_rsp->retry_delay_timer & 0xC000;
+ /* Lower 14 bits */
+ qualifier = fcp_rsp->retry_delay_timer & 0x3FFF;
+
+ if (qedf_retry_delay &&
+ scope > 0 && qualifier > 0 &&
+ qualifier <= 0x3FEF) {
+ /* Check we don't go over the max */
+ if (qualifier > QEDF_RETRY_DELAY_MAX)
+ qualifier =
+ QEDF_RETRY_DELAY_MAX;
+ fcport->retry_delay_timestamp =
+ jiffies + (qualifier * HZ / 10);
+ }
+ }
+ }
+ if (io_req->fcp_resid)
+ scsi_set_resid(sc_cmd, io_req->fcp_resid);
+ break;
+ default:
+ QEDF_INFO(&(qedf->dbg_ctx), QEDF_LOG_IO, "fcp_status=%d.\n",
+ io_req->fcp_status);
+ break;
+ }
+
+out:
+ if (qedf_io_tracing)
+ qedf_trace_io(fcport, io_req, QEDF_IO_TRACE_RSP);
+
+ io_req->sc_cmd = NULL;
+ sc_cmd->SCp.ptr = NULL;
+ sc_cmd->scsi_done(sc_cmd);
+ kref_put(&io_req->refcount, qedf_release_cmd);
+}
+
+/* Return a SCSI command in some other context besides a normal completion */
+void qedf_scsi_done(struct qedf_ctx *qedf, struct qedf_ioreq *io_req,
+ int result)
+{
+ u16 xid;
+ struct scsi_cmnd *sc_cmd;
+ int refcount;
+
+ if (!io_req)
+ return;
+
+ xid = io_req->xid;
+ sc_cmd = io_req->sc_cmd;
+
+ if (!sc_cmd) {
+ QEDF_WARN(&(qedf->dbg_ctx), "sc_cmd is NULL!\n");
+ return;
+ }
+
+ if (!sc_cmd->SCp.ptr) {
+ QEDF_WARN(&(qedf->dbg_ctx), "SCp.ptr is NULL, returned in "
+ "another context.\n");
+ return;
+ }
+
+ qedf_unmap_sg_list(qedf, io_req);
+
+ sc_cmd->result = result << 16;
+ refcount = kref_read(&io_req->refcount);
+ QEDF_INFO(&(qedf->dbg_ctx), QEDF_LOG_IO, "%d:0:%d:%d: Completing "
+ "sc_cmd=%p result=0x%08x op=0x%02x lba=0x%02x%02x%02x%02x, "
+ "allowed=%d retries=%d refcount=%d.\n",
+ qedf->lport->host->host_no, sc_cmd->device->id,
+ sc_cmd->device->lun, sc_cmd, sc_cmd->result, sc_cmd->cmnd[0],
+ sc_cmd->cmnd[2], sc_cmd->cmnd[3], sc_cmd->cmnd[4],
+ sc_cmd->cmnd[5], sc_cmd->allowed, sc_cmd->retries,
+ refcount);
+
+ /*
+ * Set resid to the whole buffer length so we won't try to resue any
+ * previously read data
+ */
+ scsi_set_resid(sc_cmd, scsi_bufflen(sc_cmd));
+
+ if (qedf_io_tracing)
+ qedf_trace_io(io_req->fcport, io_req, QEDF_IO_TRACE_RSP);
+
+ io_req->sc_cmd = NULL;
+ sc_cmd->SCp.ptr = NULL;
+ sc_cmd->scsi_done(sc_cmd);
+ kref_put(&io_req->refcount, qedf_release_cmd);
+}
+
+/*
+ * Handle warning type CQE completions. This is mainly used for REC timer
+ * popping.
+ */
+void qedf_process_warning_compl(struct qedf_ctx *qedf, struct fcoe_cqe *cqe,
+ struct qedf_ioreq *io_req)
+{
+ int rval, i;
+ struct qedf_rport *fcport = io_req->fcport;
+ u64 err_warn_bit_map;
+ u8 err_warn = 0xff;
+
+ if (!cqe)
+ return;
+
+ QEDF_ERR(&(io_req->fcport->qedf->dbg_ctx), "Warning CQE, "
+ "xid=0x%x\n", io_req->xid);
+ QEDF_ERR(&(io_req->fcport->qedf->dbg_ctx),
+ "err_warn_bitmap=%08x:%08x\n",
+ le32_to_cpu(cqe->cqe_info.err_info.err_warn_bitmap_hi),
+ le32_to_cpu(cqe->cqe_info.err_info.err_warn_bitmap_lo));
+ QEDF_ERR(&(io_req->fcport->qedf->dbg_ctx), "tx_buff_off=%08x, "
+ "rx_buff_off=%08x, rx_id=%04x\n",
+ le32_to_cpu(cqe->cqe_info.err_info.tx_buf_off),
+ le32_to_cpu(cqe->cqe_info.err_info.rx_buf_off),
+ le32_to_cpu(cqe->cqe_info.err_info.rx_id));
+
+ /* Normalize the error bitmap value to an just an unsigned int */
+ err_warn_bit_map = (u64)
+ ((u64)cqe->cqe_info.err_info.err_warn_bitmap_hi << 32) |
+ (u64)cqe->cqe_info.err_info.err_warn_bitmap_lo;
+ for (i = 0; i < 64; i++) {
+ if (err_warn_bit_map & (u64)((u64)1 << i)) {
+ err_warn = i;
+ break;
+ }
+ }
+
+ /* Check if REC TOV expired if this is a tape device */
+ if (fcport->dev_type == QEDF_RPORT_TYPE_TAPE) {
+ if (err_warn ==
+ FCOE_WARNING_CODE_REC_TOV_TIMER_EXPIRATION) {
+ QEDF_ERR(&(qedf->dbg_ctx), "REC timer expired.\n");
+ if (!test_bit(QEDF_CMD_SRR_SENT, &io_req->flags)) {
+ io_req->rx_buf_off =
+ cqe->cqe_info.err_info.rx_buf_off;
+ io_req->tx_buf_off =
+ cqe->cqe_info.err_info.tx_buf_off;
+ io_req->rx_id = cqe->cqe_info.err_info.rx_id;
+ rval = qedf_send_rec(io_req);
+ /*
+ * We only want to abort the io_req if we
+ * can't queue the REC command as we want to
+ * keep the exchange open for recovery.
+ */
+ if (rval)
+ goto send_abort;
+ }
+ return;
+ }
+ }
+
+send_abort:
+ init_completion(&io_req->abts_done);
+ rval = qedf_initiate_abts(io_req, true);
+ if (rval)
+ QEDF_ERR(&(qedf->dbg_ctx), "Failed to queue ABTS.\n");
+}
+
+/* Cleanup a command when we receive an error detection completion */
+void qedf_process_error_detect(struct qedf_ctx *qedf, struct fcoe_cqe *cqe,
+ struct qedf_ioreq *io_req)
+{
+ int rval;
+
+ if (!cqe)
+ return;
+
+ QEDF_ERR(&(io_req->fcport->qedf->dbg_ctx), "Error detection CQE, "
+ "xid=0x%x\n", io_req->xid);
+ QEDF_ERR(&(io_req->fcport->qedf->dbg_ctx),
+ "err_warn_bitmap=%08x:%08x\n",
+ le32_to_cpu(cqe->cqe_info.err_info.err_warn_bitmap_hi),
+ le32_to_cpu(cqe->cqe_info.err_info.err_warn_bitmap_lo));
+ QEDF_ERR(&(io_req->fcport->qedf->dbg_ctx), "tx_buff_off=%08x, "
+ "rx_buff_off=%08x, rx_id=%04x\n",
+ le32_to_cpu(cqe->cqe_info.err_info.tx_buf_off),
+ le32_to_cpu(cqe->cqe_info.err_info.rx_buf_off),
+ le32_to_cpu(cqe->cqe_info.err_info.rx_id));
+
+ if (qedf->stop_io_on_error) {
+ qedf_stop_all_io(qedf);
+ return;
+ }
+
+ init_completion(&io_req->abts_done);
+ rval = qedf_initiate_abts(io_req, true);
+ if (rval)
+ QEDF_ERR(&(qedf->dbg_ctx), "Failed to queue ABTS.\n");
+}
+
+static void qedf_flush_els_req(struct qedf_ctx *qedf,
+ struct qedf_ioreq *els_req)
+{
+ QEDF_INFO(&(qedf->dbg_ctx), QEDF_LOG_IO,
+ "Flushing ELS request xid=0x%x refcount=%d.\n", els_req->xid,
+ kref_read(&els_req->refcount));
+
+ /*
+ * Need to distinguish this from a timeout when calling the
+ * els_req->cb_func.
+ */
+ els_req->event = QEDF_IOREQ_EV_ELS_FLUSH;
+
+ /* Cancel the timer */
+ cancel_delayed_work_sync(&els_req->timeout_work);
+
+ /* Call callback function to complete command */
+ if (els_req->cb_func && els_req->cb_arg) {
+ els_req->cb_func(els_req->cb_arg);
+ els_req->cb_arg = NULL;
+ }
+
+ /* Release kref for original initiate_els */
+ kref_put(&els_req->refcount, qedf_release_cmd);
+}
+
+/* A value of -1 for lun is a wild card that means flush all
+ * active SCSI I/Os for the target.
+ */
+void qedf_flush_active_ios(struct qedf_rport *fcport, int lun)
+{
+ struct qedf_ioreq *io_req;
+ struct qedf_ctx *qedf;
+ struct qedf_cmd_mgr *cmd_mgr;
+ int i, rc;
+
+ if (!fcport)
+ return;
+
+ qedf = fcport->qedf;
+ cmd_mgr = qedf->cmd_mgr;
+
+ QEDF_INFO(&(qedf->dbg_ctx), QEDF_LOG_IO, "Flush active i/o's.\n");
+
+ for (i = 0; i < FCOE_PARAMS_NUM_TASKS; i++) {
+ io_req = &cmd_mgr->cmds[i];
+
+ if (!io_req)
+ continue;
+ if (io_req->fcport != fcport)
+ continue;
+ if (io_req->cmd_type == QEDF_ELS) {
+ rc = kref_get_unless_zero(&io_req->refcount);
+ if (!rc) {
+ QEDF_ERR(&(qedf->dbg_ctx),
+ "Could not get kref for io_req=0x%p.\n",
+ io_req);
+ continue;
+ }
+ qedf_flush_els_req(qedf, io_req);
+ /*
+ * Release the kref and go back to the top of the
+ * loop.
+ */
+ goto free_cmd;
+ }
+
+ if (!io_req->sc_cmd)
+ continue;
+ if (lun > 0) {
+ if (io_req->sc_cmd->device->lun !=
+ (u64)lun)
+ continue;
+ }
+
+ /*
+ * Use kref_get_unless_zero in the unlikely case the command
+ * we're about to flush was completed in the normal SCSI path
+ */
+ rc = kref_get_unless_zero(&io_req->refcount);
+ if (!rc) {
+ QEDF_ERR(&(qedf->dbg_ctx), "Could not get kref for "
+ "io_req=0x%p\n", io_req);
+ continue;
+ }
+ QEDF_INFO(&(qedf->dbg_ctx), QEDF_LOG_IO,
+ "Cleanup xid=0x%x.\n", io_req->xid);
+
+ /* Cleanup task and return I/O mid-layer */
+ qedf_initiate_cleanup(io_req, true);
+
+free_cmd:
+ kref_put(&io_req->refcount, qedf_release_cmd);
+ }
+}
+
+/*
+ * Initiate a ABTS middle path command. Note that we don't have to initialize
+ * the task context for an ABTS task.
+ */
+int qedf_initiate_abts(struct qedf_ioreq *io_req, bool return_scsi_cmd_on_abts)
+{
+ struct fc_lport *lport;
+ struct qedf_rport *fcport = io_req->fcport;
+ struct fc_rport_priv *rdata = fcport->rdata;
+ struct qedf_ctx *qedf = fcport->qedf;
+ u16 xid;
+ u32 r_a_tov = 0;
+ int rc = 0;
+ unsigned long flags;
+
+ r_a_tov = rdata->r_a_tov;
+ lport = qedf->lport;
+
+ if (!test_bit(QEDF_RPORT_SESSION_READY, &fcport->flags)) {
+ QEDF_ERR(&(qedf->dbg_ctx), "tgt not offloaded\n");
+ rc = 1;
+ goto abts_err;
+ }
+
+ if (lport->state != LPORT_ST_READY || !(lport->link_up)) {
+ QEDF_ERR(&(qedf->dbg_ctx), "link is not ready\n");
+ rc = 1;
+ goto abts_err;
+ }
+
+ if (atomic_read(&qedf->link_down_tmo_valid) > 0) {
+ QEDF_ERR(&(qedf->dbg_ctx), "link_down_tmo active.\n");
+ rc = 1;
+ goto abts_err;
+ }
+
+ /* Ensure room on SQ */
+ if (!atomic_read(&fcport->free_sqes)) {
+ QEDF_ERR(&(qedf->dbg_ctx), "No SQ entries available\n");
+ rc = 1;
+ goto abts_err;
+ }
+
+
+ kref_get(&io_req->refcount);
+
+ xid = io_req->xid;
+ qedf->control_requests++;
+ qedf->packet_aborts++;
+
+ /* Set the return CPU to be the same as the request one */
+ io_req->cpu = smp_processor_id();
+
+ /* Set the command type to abort */
+ io_req->cmd_type = QEDF_ABTS;
+ io_req->return_scsi_cmd_on_abts = return_scsi_cmd_on_abts;
+
+ set_bit(QEDF_CMD_IN_ABORT, &io_req->flags);
+ QEDF_INFO(&(qedf->dbg_ctx), QEDF_LOG_SCSI_TM, "ABTS io_req xid = "
+ "0x%x\n", xid);
+
+ qedf_cmd_timer_set(qedf, io_req, QEDF_ABORT_TIMEOUT * HZ);
+
+ spin_lock_irqsave(&fcport->rport_lock, flags);
+
+ /* Add ABTS to send queue */
+ qedf_add_to_sq(fcport, xid, 0, FCOE_TASK_TYPE_ABTS, 0);
+
+ /* Ring doorbell */
+ qedf_ring_doorbell(fcport);
+
+ spin_unlock_irqrestore(&fcport->rport_lock, flags);
+
+ return rc;
+abts_err:
+ /*
+ * If the ABTS task fails to queue then we need to cleanup the
+ * task at the firmware.
+ */
+ qedf_initiate_cleanup(io_req, return_scsi_cmd_on_abts);
+ return rc;
+}
+
+void qedf_process_abts_compl(struct qedf_ctx *qedf, struct fcoe_cqe *cqe,
+ struct qedf_ioreq *io_req)
+{
+ uint32_t r_ctl;
+ uint16_t xid;
+
+ QEDF_INFO(&(qedf->dbg_ctx), QEDF_LOG_SCSI_TM, "Entered with xid = "
+ "0x%x cmd_type = %d\n", io_req->xid, io_req->cmd_type);
+
+ cancel_delayed_work(&io_req->timeout_work);
+
+ xid = io_req->xid;
+ r_ctl = cqe->cqe_info.abts_info.r_ctl;
+
+ switch (r_ctl) {
+ case FC_RCTL_BA_ACC:
+ QEDF_INFO(&(qedf->dbg_ctx), QEDF_LOG_SCSI_TM,
+ "ABTS response - ACC Send RRQ after R_A_TOV\n");
+ io_req->event = QEDF_IOREQ_EV_ABORT_SUCCESS;
+ /*
+ * Dont release this cmd yet. It will be relesed
+ * after we get RRQ response
+ */
+ kref_get(&io_req->refcount);
+ queue_delayed_work(qedf->dpc_wq, &io_req->rrq_work,
+ msecs_to_jiffies(qedf->lport->r_a_tov));
+ break;
+ /* For error cases let the cleanup return the command */
+ case FC_RCTL_BA_RJT:
+ QEDF_INFO(&(qedf->dbg_ctx), QEDF_LOG_SCSI_TM,
+ "ABTS response - RJT\n");
+ io_req->event = QEDF_IOREQ_EV_ABORT_FAILED;
+ break;
+ default:
+ QEDF_ERR(&(qedf->dbg_ctx), "Unknown ABTS response\n");
+ break;
+ }
+
+ clear_bit(QEDF_CMD_IN_ABORT, &io_req->flags);
+
+ if (io_req->sc_cmd) {
+ if (io_req->return_scsi_cmd_on_abts)
+ qedf_scsi_done(qedf, io_req, DID_ERROR);
+ }
+
+ /* Notify eh_abort handler that ABTS is complete */
+ complete(&io_req->abts_done);
+
+ kref_put(&io_req->refcount, qedf_release_cmd);
+}
+
+int qedf_init_mp_req(struct qedf_ioreq *io_req)
+{
+ struct qedf_mp_req *mp_req;
+ struct fcoe_sge *mp_req_bd;
+ struct fcoe_sge *mp_resp_bd;
+ struct qedf_ctx *qedf = io_req->fcport->qedf;
+ dma_addr_t addr;
+ uint64_t sz;
+
+ QEDF_INFO(&(qedf->dbg_ctx), QEDF_LOG_MP_REQ, "Entered.\n");
+
+ mp_req = (struct qedf_mp_req *)&(io_req->mp_req);
+ memset(mp_req, 0, sizeof(struct qedf_mp_req));
+
+ if (io_req->cmd_type != QEDF_ELS) {
+ mp_req->req_len = sizeof(struct fcp_cmnd);
+ io_req->data_xfer_len = mp_req->req_len;
+ } else
+ mp_req->req_len = io_req->data_xfer_len;
+
+ mp_req->req_buf = dma_alloc_coherent(&qedf->pdev->dev, QEDF_PAGE_SIZE,
+ &mp_req->req_buf_dma, GFP_KERNEL);
+ if (!mp_req->req_buf) {
+ QEDF_ERR(&(qedf->dbg_ctx), "Unable to alloc MP req buffer\n");
+ qedf_free_mp_resc(io_req);
+ return -ENOMEM;
+ }
+
+ mp_req->resp_buf = dma_alloc_coherent(&qedf->pdev->dev,
+ QEDF_PAGE_SIZE, &mp_req->resp_buf_dma, GFP_KERNEL);
+ if (!mp_req->resp_buf) {
+ QEDF_ERR(&(qedf->dbg_ctx), "Unable to alloc TM resp "
+ "buffer\n");
+ qedf_free_mp_resc(io_req);
+ return -ENOMEM;
+ }
+
+ /* Allocate and map mp_req_bd and mp_resp_bd */
+ sz = sizeof(struct fcoe_sge);
+ mp_req->mp_req_bd = dma_alloc_coherent(&qedf->pdev->dev, sz,
+ &mp_req->mp_req_bd_dma, GFP_KERNEL);
+ if (!mp_req->mp_req_bd) {
+ QEDF_ERR(&(qedf->dbg_ctx), "Unable to alloc MP req bd\n");
+ qedf_free_mp_resc(io_req);
+ return -ENOMEM;
+ }
+
+ mp_req->mp_resp_bd = dma_alloc_coherent(&qedf->pdev->dev, sz,
+ &mp_req->mp_resp_bd_dma, GFP_KERNEL);
+ if (!mp_req->mp_resp_bd) {
+ QEDF_ERR(&(qedf->dbg_ctx), "Unable to alloc MP resp bd\n");
+ qedf_free_mp_resc(io_req);
+ return -ENOMEM;
+ }
+
+ /* Fill bd table */
+ addr = mp_req->req_buf_dma;
+ mp_req_bd = mp_req->mp_req_bd;
+ mp_req_bd->sge_addr.lo = U64_LO(addr);
+ mp_req_bd->sge_addr.hi = U64_HI(addr);
+ mp_req_bd->size = QEDF_PAGE_SIZE;
+
+ /*
+ * MP buffer is either a task mgmt command or an ELS.
+ * So the assumption is that it consumes a single bd
+ * entry in the bd table
+ */
+ mp_resp_bd = mp_req->mp_resp_bd;
+ addr = mp_req->resp_buf_dma;
+ mp_resp_bd->sge_addr.lo = U64_LO(addr);
+ mp_resp_bd->sge_addr.hi = U64_HI(addr);
+ mp_resp_bd->size = QEDF_PAGE_SIZE;
+
+ return 0;
+}
+
+/*
+ * Last ditch effort to clear the port if it's stuck. Used only after a
+ * cleanup task times out.
+ */
+static void qedf_drain_request(struct qedf_ctx *qedf)
+{
+ if (test_bit(QEDF_DRAIN_ACTIVE, &qedf->flags)) {
+ QEDF_ERR(&(qedf->dbg_ctx), "MCP drain already active.\n");
+ return;
+ }
+
+ /* Set bit to return all queuecommand requests as busy */
+ set_bit(QEDF_DRAIN_ACTIVE, &qedf->flags);
+
+ /* Call qed drain request for function. Should be synchronous */
+ qed_ops->common->drain(qedf->cdev);
+
+ /* Settle time for CQEs to be returned */
+ msleep(100);
+
+ /* Unplug and continue */
+ clear_bit(QEDF_DRAIN_ACTIVE, &qedf->flags);
+}
+
+/*
+ * Returns SUCCESS if the cleanup task does not timeout, otherwise return
+ * FAILURE.
+ */
+int qedf_initiate_cleanup(struct qedf_ioreq *io_req,
+ bool return_scsi_cmd_on_abts)
+{
+ struct qedf_rport *fcport;
+ struct qedf_ctx *qedf;
+ uint16_t xid;
+ struct fcoe_task_context *task;
+ int tmo = 0;
+ int rc = SUCCESS;
+ unsigned long flags;
+
+ fcport = io_req->fcport;
+ if (!fcport) {
+ QEDF_ERR(NULL, "fcport is NULL.\n");
+ return SUCCESS;
+ }
+
+ qedf = fcport->qedf;
+ if (!qedf) {
+ QEDF_ERR(NULL, "qedf is NULL.\n");
+ return SUCCESS;
+ }
+
+ if (!test_bit(QEDF_CMD_OUTSTANDING, &io_req->flags) ||
+ test_bit(QEDF_CMD_IN_CLEANUP, &io_req->flags)) {
+ QEDF_ERR(&(qedf->dbg_ctx), "io_req xid=0x%x already in "
+ "cleanup processing or already completed.\n",
+ io_req->xid);
+ return SUCCESS;
+ }
+
+ /* Ensure room on SQ */
+ if (!atomic_read(&fcport->free_sqes)) {
+ QEDF_ERR(&(qedf->dbg_ctx), "No SQ entries available\n");
+ return FAILED;
+ }
+
+
+ QEDF_INFO(&(qedf->dbg_ctx), QEDF_LOG_IO, "Entered xid=0x%x\n",
+ io_req->xid);
+
+ /* Cleanup cmds re-use the same TID as the original I/O */
+ xid = io_req->xid;
+ io_req->cmd_type = QEDF_CLEANUP;
+ io_req->return_scsi_cmd_on_abts = return_scsi_cmd_on_abts;
+
+ /* Set the return CPU to be the same as the request one */
+ io_req->cpu = smp_processor_id();
+
+ set_bit(QEDF_CMD_IN_CLEANUP, &io_req->flags);
+
+ task = qedf_get_task_mem(&qedf->tasks, xid);
+
+ init_completion(&io_req->tm_done);
+
+ /* Obtain free SQ entry */
+ spin_lock_irqsave(&fcport->rport_lock, flags);
+ qedf_add_to_sq(fcport, xid, 0, FCOE_TASK_TYPE_EXCHANGE_CLEANUP, 0);
+
+ /* Ring doorbell */
+ qedf_ring_doorbell(fcport);
+ spin_unlock_irqrestore(&fcport->rport_lock, flags);
+
+ tmo = wait_for_completion_timeout(&io_req->tm_done,
+ QEDF_CLEANUP_TIMEOUT * HZ);
+
+ if (!tmo) {
+ rc = FAILED;
+ /* Timeout case */
+ QEDF_ERR(&(qedf->dbg_ctx), "Cleanup command timeout, "
+ "xid=%x.\n", io_req->xid);
+ clear_bit(QEDF_CMD_IN_CLEANUP, &io_req->flags);
+ /* Issue a drain request if cleanup task times out */
+ QEDF_ERR(&(qedf->dbg_ctx), "Issuing MCP drain request.\n");
+ qedf_drain_request(qedf);
+ }
+
+ if (io_req->sc_cmd) {
+ if (io_req->return_scsi_cmd_on_abts)
+ qedf_scsi_done(qedf, io_req, DID_ERROR);
+ }
+
+ if (rc == SUCCESS)
+ io_req->event = QEDF_IOREQ_EV_CLEANUP_SUCCESS;
+ else
+ io_req->event = QEDF_IOREQ_EV_CLEANUP_FAILED;
+
+ return rc;
+}
+
+void qedf_process_cleanup_compl(struct qedf_ctx *qedf, struct fcoe_cqe *cqe,
+ struct qedf_ioreq *io_req)
+{
+ QEDF_INFO(&(qedf->dbg_ctx), QEDF_LOG_IO, "Entered xid = 0x%x\n",
+ io_req->xid);
+
+ clear_bit(QEDF_CMD_IN_CLEANUP, &io_req->flags);
+
+ /* Complete so we can finish cleaning up the I/O */
+ complete(&io_req->tm_done);
+}
+
+static int qedf_execute_tmf(struct qedf_rport *fcport, struct scsi_cmnd *sc_cmd,
+ uint8_t tm_flags)
+{
+ struct qedf_ioreq *io_req;
+ struct qedf_mp_req *tm_req;
+ struct fcoe_task_context *task;
+ struct fc_frame_header *fc_hdr;
+ struct fcp_cmnd *fcp_cmnd;
+ struct qedf_ctx *qedf = fcport->qedf;
+ int rc = 0;
+ uint16_t xid;
+ uint32_t sid, did;
+ int tmo = 0;
+ unsigned long flags;
+
+ if (!sc_cmd) {
+ QEDF_ERR(&(qedf->dbg_ctx), "invalid arg\n");
+ return FAILED;
+ }
+
+ if (!(test_bit(QEDF_RPORT_SESSION_READY, &fcport->flags))) {
+ QEDF_ERR(&(qedf->dbg_ctx), "fcport not offloaded\n");
+ rc = FAILED;
+ return FAILED;
+ }
+
+ QEDF_INFO(&(qedf->dbg_ctx), QEDF_LOG_SCSI_TM, "portid = 0x%x "
+ "tm_flags = %d\n", fcport->rdata->ids.port_id, tm_flags);
+
+ io_req = qedf_alloc_cmd(fcport, QEDF_TASK_MGMT_CMD);
+ if (!io_req) {
+ QEDF_ERR(&(qedf->dbg_ctx), "Failed TMF");
+ rc = -EAGAIN;
+ goto reset_tmf_err;
+ }
+
+ /* Initialize rest of io_req fields */
+ io_req->sc_cmd = sc_cmd;
+ io_req->fcport = fcport;
+ io_req->cmd_type = QEDF_TASK_MGMT_CMD;
+
+ /* Set the return CPU to be the same as the request one */
+ io_req->cpu = smp_processor_id();
+
+ tm_req = (struct qedf_mp_req *)&(io_req->mp_req);
+
+ rc = qedf_init_mp_req(io_req);
+ if (rc == FAILED) {
+ QEDF_ERR(&(qedf->dbg_ctx), "Task mgmt MP request init "
+ "failed\n");
+ kref_put(&io_req->refcount, qedf_release_cmd);
+ goto reset_tmf_err;
+ }
+
+ /* Set TM flags */
+ io_req->io_req_flags = 0;
+ tm_req->tm_flags = tm_flags;
+
+ /* Default is to return a SCSI command when an error occurs */
+ io_req->return_scsi_cmd_on_abts = true;
+
+ /* Fill FCP_CMND */
+ qedf_build_fcp_cmnd(io_req, (struct fcp_cmnd *)tm_req->req_buf);
+ fcp_cmnd = (struct fcp_cmnd *)tm_req->req_buf;
+ memset(fcp_cmnd->fc_cdb, 0, FCP_CMND_LEN);
+ fcp_cmnd->fc_dl = 0;
+
+ /* Fill FC header */
+ fc_hdr = &(tm_req->req_fc_hdr);
+ sid = fcport->sid;
+ did = fcport->rdata->ids.port_id;
+ __fc_fill_fc_hdr(fc_hdr, FC_RCTL_DD_UNSOL_CMD, sid, did,
+ FC_TYPE_FCP, FC_FC_FIRST_SEQ | FC_FC_END_SEQ |
+ FC_FC_SEQ_INIT, 0);
+ /* Obtain exchange id */
+ xid = io_req->xid;
+
+ QEDF_INFO(&(qedf->dbg_ctx), QEDF_LOG_SCSI_TM, "TMF io_req xid = "
+ "0x%x\n", xid);
+
+ /* Initialize task context for this IO request */
+ task = qedf_get_task_mem(&qedf->tasks, xid);
+ qedf_init_mp_task(io_req, task);
+
+ init_completion(&io_req->tm_done);
+
+ /* Obtain free SQ entry */
+ spin_lock_irqsave(&fcport->rport_lock, flags);
+ qedf_add_to_sq(fcport, xid, 0, FCOE_TASK_TYPE_MIDPATH, 0);
+
+ /* Ring doorbell */
+ qedf_ring_doorbell(fcport);
+ spin_unlock_irqrestore(&fcport->rport_lock, flags);
+
+ tmo = wait_for_completion_timeout(&io_req->tm_done,
+ QEDF_TM_TIMEOUT * HZ);
+
+ if (!tmo) {
+ rc = FAILED;
+ QEDF_ERR(&(qedf->dbg_ctx), "wait for tm_cmpl timeout!\n");
+ } else {
+ /* Check TMF response code */
+ if (io_req->fcp_rsp_code == 0)
+ rc = SUCCESS;
+ else
+ rc = FAILED;
+ }
+
+ if (tm_flags == FCP_TMF_LUN_RESET)
+ qedf_flush_active_ios(fcport, (int)sc_cmd->device->lun);
+ else
+ qedf_flush_active_ios(fcport, -1);
+
+ kref_put(&io_req->refcount, qedf_release_cmd);
+
+ if (rc != SUCCESS) {
+ QEDF_ERR(&(qedf->dbg_ctx), "task mgmt command failed...\n");
+ rc = FAILED;
+ } else {
+ QEDF_ERR(&(qedf->dbg_ctx), "task mgmt command success...\n");
+ rc = SUCCESS;
+ }
+reset_tmf_err:
+ return rc;
+}
+
+int qedf_initiate_tmf(struct scsi_cmnd *sc_cmd, u8 tm_flags)
+{
+ struct fc_rport *rport = starget_to_rport(scsi_target(sc_cmd->device));
+ struct fc_rport_libfc_priv *rp = rport->dd_data;
+ struct qedf_rport *fcport = (struct qedf_rport *)&rp[1];
+ struct qedf_ctx *qedf;
+ struct fc_lport *lport;
+ int rc = SUCCESS;
+ int rval;
+
+ rval = fc_remote_port_chkready(rport);
+
+ if (rval) {
+ QEDF_ERR(NULL, "device_reset rport not ready\n");
+ rc = FAILED;
+ goto tmf_err;
+ }
+
+ if (fcport == NULL) {
+ QEDF_ERR(NULL, "device_reset: rport is NULL\n");
+ rc = FAILED;
+ goto tmf_err;
+ }
+
+ qedf = fcport->qedf;
+ lport = qedf->lport;
+
+ if (test_bit(QEDF_UNLOADING, &qedf->flags) ||
+ test_bit(QEDF_DBG_STOP_IO, &qedf->flags)) {
+ rc = SUCCESS;
+ goto tmf_err;
+ }
+
+ if (lport->state != LPORT_ST_READY || !(lport->link_up)) {
+ QEDF_ERR(&(qedf->dbg_ctx), "link is not ready\n");
+ rc = FAILED;
+ goto tmf_err;
+ }
+
+ rc = qedf_execute_tmf(fcport, sc_cmd, tm_flags);
+
+tmf_err:
+ return rc;
+}
+
+void qedf_process_tmf_compl(struct qedf_ctx *qedf, struct fcoe_cqe *cqe,
+ struct qedf_ioreq *io_req)
+{
+ struct fcoe_cqe_rsp_info *fcp_rsp;
+ struct fcoe_cqe_midpath_info *mp_info;
+
+
+ /* Get TMF response length from CQE */
+ mp_info = &cqe->cqe_info.midpath_info;
+ io_req->mp_req.resp_len = mp_info->data_placement_size;
+ QEDF_INFO(&(qedf->dbg_ctx), QEDF_LOG_SCSI_TM,
+ "Response len is %d.\n", io_req->mp_req.resp_len);
+
+ fcp_rsp = &cqe->cqe_info.rsp_info;
+ qedf_parse_fcp_rsp(io_req, fcp_rsp);
+
+ io_req->sc_cmd = NULL;
+ complete(&io_req->tm_done);
+}
+
+void qedf_process_unsol_compl(struct qedf_ctx *qedf, uint16_t que_idx,
+ struct fcoe_cqe *cqe)
+{
+ unsigned long flags;
+ uint16_t tmp;
+ uint16_t pktlen = cqe->cqe_info.unsolic_info.pkt_len;
+ u32 payload_len, crc;
+ struct fc_frame_header *fh;
+ struct fc_frame *fp;
+ struct qedf_io_work *io_work;
+ u32 bdq_idx;
+ void *bdq_addr;
+
+ QEDF_INFO(&(qedf->dbg_ctx), QEDF_LOG_UNSOL,
+ "address.hi=%x address.lo=%x opaque_data.hi=%x "
+ "opaque_data.lo=%x bdq_prod_idx=%u len=%u.\n",
+ le32_to_cpu(cqe->cqe_info.unsolic_info.bd_info.address.hi),
+ le32_to_cpu(cqe->cqe_info.unsolic_info.bd_info.address.lo),
+ le32_to_cpu(cqe->cqe_info.unsolic_info.bd_info.opaque.hi),
+ le32_to_cpu(cqe->cqe_info.unsolic_info.bd_info.opaque.lo),
+ qedf->bdq_prod_idx, pktlen);
+
+ bdq_idx = le32_to_cpu(cqe->cqe_info.unsolic_info.bd_info.opaque.lo);
+ if (bdq_idx >= QEDF_BDQ_SIZE) {
+ QEDF_ERR(&(qedf->dbg_ctx), "bdq_idx is out of range %d.\n",
+ bdq_idx);
+ goto increment_prod;
+ }
+
+ bdq_addr = qedf->bdq[bdq_idx].buf_addr;
+ if (!bdq_addr) {
+ QEDF_ERR(&(qedf->dbg_ctx), "bdq_addr is NULL, dropping "
+ "unsolicited packet.\n");
+ goto increment_prod;
+ }
+
+ if (qedf_dump_frames) {
+ QEDF_INFO(&(qedf->dbg_ctx), QEDF_LOG_UNSOL,
+ "BDQ frame is at addr=%p.\n", bdq_addr);
+ print_hex_dump(KERN_WARNING, "bdq ", DUMP_PREFIX_OFFSET, 16, 1,
+ (void *)bdq_addr, pktlen, false);
+ }
+
+ /* Allocate frame */
+ payload_len = pktlen - sizeof(struct fc_frame_header);
+ fp = fc_frame_alloc(qedf->lport, payload_len);
+ if (!fp) {
+ QEDF_ERR(&(qedf->dbg_ctx), "Could not allocate fp.\n");
+ goto increment_prod;
+ }
+
+ /* Copy data from BDQ buffer into fc_frame struct */
+ fh = (struct fc_frame_header *)fc_frame_header_get(fp);
+ memcpy(fh, (void *)bdq_addr, pktlen);
+
+ /* Initialize the frame so libfc sees it as a valid frame */
+ crc = fcoe_fc_crc(fp);
+ fc_frame_init(fp);
+ fr_dev(fp) = qedf->lport;
+ fr_sof(fp) = FC_SOF_I3;
+ fr_eof(fp) = FC_EOF_T;
+ fr_crc(fp) = cpu_to_le32(~crc);
+
+ /*
+ * We need to return the frame back up to libfc in a non-atomic
+ * context
+ */
+ io_work = mempool_alloc(qedf->io_mempool, GFP_ATOMIC);
+ if (!io_work) {
+ QEDF_WARN(&(qedf->dbg_ctx), "Could not allocate "
+ "work for I/O completion.\n");
+ fc_frame_free(fp);
+ goto increment_prod;
+ }
+ memset(io_work, 0, sizeof(struct qedf_io_work));
+
+ INIT_WORK(&io_work->work, qedf_fp_io_handler);
+
+ /* Copy contents of CQE for deferred processing */
+ memcpy(&io_work->cqe, cqe, sizeof(struct fcoe_cqe));
+
+ io_work->qedf = qedf;
+ io_work->fp = fp;
+
+ queue_work_on(smp_processor_id(), qedf_io_wq, &io_work->work);
+increment_prod:
+ spin_lock_irqsave(&qedf->hba_lock, flags);
+
+ /* Increment producer to let f/w know we've handled the frame */
+ qedf->bdq_prod_idx++;
+
+ /* Producer index wraps at uint16_t boundary */
+ if (qedf->bdq_prod_idx == 0xffff)
+ qedf->bdq_prod_idx = 0;
+
+ writew(qedf->bdq_prod_idx, qedf->bdq_primary_prod);
+ tmp = readw(qedf->bdq_primary_prod);
+ writew(qedf->bdq_prod_idx, qedf->bdq_secondary_prod);
+ tmp = readw(qedf->bdq_secondary_prod);
+
+ spin_unlock_irqrestore(&qedf->hba_lock, flags);
+}
--- /dev/null
+/*
+ * QLogic FCoE Offload Driver
+ * Copyright (c) 2016 Cavium Inc.
+ *
+ * This software is available under the terms of the GNU General Public License
+ * (GPL) Version 2, available from the file COPYING in the main directory of
+ * this source tree.
+ */
+#include <linux/init.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/pci.h>
+#include <linux/device.h>
+#include <linux/highmem.h>
+#include <linux/crc32.h>
+#include <linux/interrupt.h>
+#include <linux/list.h>
+#include <linux/kthread.h>
+#include <scsi/libfc.h>
+#include <scsi/scsi_host.h>
+#include <linux/if_ether.h>
+#include <linux/if_vlan.h>
+#include <linux/cpu.h>
+#include "qedf.h"
+
+const struct qed_fcoe_ops *qed_ops;
+
+static int qedf_probe(struct pci_dev *pdev, const struct pci_device_id *id);
+static void qedf_remove(struct pci_dev *pdev);
+
+extern struct qedf_debugfs_ops qedf_debugfs_ops;
+extern struct file_operations qedf_dbg_fops;
+
+/*
+ * Driver module parameters.
+ */
+static unsigned int qedf_dev_loss_tmo = 60;
+module_param_named(dev_loss_tmo, qedf_dev_loss_tmo, int, S_IRUGO);
+MODULE_PARM_DESC(dev_loss_tmo, " dev_loss_tmo setting for attached "
+ "remote ports (default 60)");
+
+uint qedf_debug = QEDF_LOG_INFO;
+module_param_named(debug, qedf_debug, uint, S_IRUGO);
+MODULE_PARM_DESC(qedf_debug, " Debug mask. Pass '1' to enable default debugging"
+ " mask");
+
+static uint qedf_fipvlan_retries = 30;
+module_param_named(fipvlan_retries, qedf_fipvlan_retries, int, S_IRUGO);
+MODULE_PARM_DESC(fipvlan_retries, " Number of FIP VLAN requests to attempt "
+ "before giving up (default 30)");
+
+static uint qedf_fallback_vlan = QEDF_FALLBACK_VLAN;
+module_param_named(fallback_vlan, qedf_fallback_vlan, int, S_IRUGO);
+MODULE_PARM_DESC(fallback_vlan, " VLAN ID to try if fip vlan request fails "
+ "(default 1002).");
+
+static uint qedf_default_prio = QEDF_DEFAULT_PRIO;
+module_param_named(default_prio, qedf_default_prio, int, S_IRUGO);
+MODULE_PARM_DESC(default_prio, " Default 802.1q priority for FIP and FCoE"
+ " traffic (default 3).");
+
+uint qedf_dump_frames;
+module_param_named(dump_frames, qedf_dump_frames, int, S_IRUGO | S_IWUSR);
+MODULE_PARM_DESC(dump_frames, " Print the skb data of FIP and FCoE frames "
+ "(default off)");
+
+static uint qedf_queue_depth;
+module_param_named(queue_depth, qedf_queue_depth, int, S_IRUGO);
+MODULE_PARM_DESC(queue_depth, " Sets the queue depth for all LUNs discovered "
+ "by the qedf driver. Default is 0 (use OS default).");
+
+uint qedf_io_tracing;
+module_param_named(io_tracing, qedf_io_tracing, int, S_IRUGO | S_IWUSR);
+MODULE_PARM_DESC(io_tracing, " Enable logging of SCSI requests/completions "
+ "into trace buffer. (default off).");
+
+static uint qedf_max_lun = MAX_FIBRE_LUNS;
+module_param_named(max_lun, qedf_max_lun, int, S_IRUGO);
+MODULE_PARM_DESC(max_lun, " Sets the maximum luns per target that the driver "
+ "supports. (default 0xffffffff)");
+
+uint qedf_link_down_tmo;
+module_param_named(link_down_tmo, qedf_link_down_tmo, int, S_IRUGO);
+MODULE_PARM_DESC(link_down_tmo, " Delays informing the fcoe transport that the "
+ "link is down by N seconds.");
+
+bool qedf_retry_delay;
+module_param_named(retry_delay, qedf_retry_delay, bool, S_IRUGO | S_IWUSR);
+MODULE_PARM_DESC(retry_delay, " Enable/disable handling of FCP_RSP IU retry "
+ "delay handling (default off).");
+
+static uint qedf_dp_module;
+module_param_named(dp_module, qedf_dp_module, uint, S_IRUGO);
+MODULE_PARM_DESC(dp_module, " bit flags control for verbose printk passed "
+ "qed module during probe.");
+
+static uint qedf_dp_level;
+module_param_named(dp_level, qedf_dp_level, uint, S_IRUGO);
+MODULE_PARM_DESC(dp_level, " printk verbosity control passed to qed module "
+ "during probe (0-3: 0 more verbose).");
+
+struct workqueue_struct *qedf_io_wq;
+
+static struct fcoe_percpu_s qedf_global;
+static DEFINE_SPINLOCK(qedf_global_lock);
+
+static struct kmem_cache *qedf_io_work_cache;
+
+void qedf_set_vlan_id(struct qedf_ctx *qedf, int vlan_id)
+{
+ qedf->vlan_id = vlan_id;
+ qedf->vlan_id |= qedf_default_prio << VLAN_PRIO_SHIFT;
+ QEDF_INFO(&(qedf->dbg_ctx), QEDF_LOG_DISC, "Setting vlan_id=%04x "
+ "prio=%d.\n", vlan_id, qedf_default_prio);
+}
+
+/* Returns true if we have a valid vlan, false otherwise */
+static bool qedf_initiate_fipvlan_req(struct qedf_ctx *qedf)
+{
+ int rc;
+
+ if (atomic_read(&qedf->link_state) != QEDF_LINK_UP) {
+ QEDF_ERR(&(qedf->dbg_ctx), "Link not up.\n");
+ return false;
+ }
+
+ while (qedf->fipvlan_retries--) {
+ if (qedf->vlan_id > 0)
+ return true;
+ QEDF_INFO(&(qedf->dbg_ctx), QEDF_LOG_DISC,
+ "Retry %d.\n", qedf->fipvlan_retries);
+ init_completion(&qedf->fipvlan_compl);
+ qedf_fcoe_send_vlan_req(qedf);
+ rc = wait_for_completion_timeout(&qedf->fipvlan_compl,
+ 1 * HZ);
+ if (rc > 0) {
+ fcoe_ctlr_link_up(&qedf->ctlr);
+ return true;
+ }
+ }
+
+ return false;
+}
+
+static void qedf_handle_link_update(struct work_struct *work)
+{
+ struct qedf_ctx *qedf =
+ container_of(work, struct qedf_ctx, link_update.work);
+ int rc;
+
+ QEDF_INFO(&(qedf->dbg_ctx), QEDF_LOG_DISC, "Entered.\n");
+
+ if (atomic_read(&qedf->link_state) == QEDF_LINK_UP) {
+ rc = qedf_initiate_fipvlan_req(qedf);
+ if (rc)
+ return;
+ /*
+ * If we get here then we never received a repsonse to our
+ * fip vlan request so set the vlan_id to the default and
+ * tell FCoE that the link is up
+ */
+ QEDF_WARN(&(qedf->dbg_ctx), "Did not receive FIP VLAN "
+ "response, falling back to default VLAN %d.\n",
+ qedf_fallback_vlan);
+ qedf_set_vlan_id(qedf, QEDF_FALLBACK_VLAN);
+
+ /*
+ * Zero out data_src_addr so we'll update it with the new
+ * lport port_id
+ */
+ eth_zero_addr(qedf->data_src_addr);
+ fcoe_ctlr_link_up(&qedf->ctlr);
+ } else if (atomic_read(&qedf->link_state) == QEDF_LINK_DOWN) {
+ /*
+ * If we hit here and link_down_tmo_valid is still 1 it means
+ * that link_down_tmo timed out so set it to 0 to make sure any
+ * other readers have accurate state.
+ */
+ atomic_set(&qedf->link_down_tmo_valid, 0);
+ QEDF_INFO(&(qedf->dbg_ctx), QEDF_LOG_DISC,
+ "Calling fcoe_ctlr_link_down().\n");
+ fcoe_ctlr_link_down(&qedf->ctlr);
+ qedf_wait_for_upload(qedf);
+ /* Reset the number of FIP VLAN retries */
+ qedf->fipvlan_retries = qedf_fipvlan_retries;
+ }
+}
+
+static void qedf_flogi_resp(struct fc_seq *seq, struct fc_frame *fp,
+ void *arg)
+{
+ struct fc_exch *exch = fc_seq_exch(seq);
+ struct fc_lport *lport = exch->lp;
+ struct qedf_ctx *qedf = lport_priv(lport);
+
+ if (!qedf) {
+ QEDF_ERR(NULL, "qedf is NULL.\n");
+ return;
+ }
+
+ /*
+ * If ERR_PTR is set then don't try to stat anything as it will cause
+ * a crash when we access fp.
+ */
+ if (IS_ERR(fp)) {
+ QEDF_INFO(&(qedf->dbg_ctx), QEDF_LOG_ELS,
+ "fp has IS_ERR() set.\n");
+ goto skip_stat;
+ }
+
+ /* Log stats for FLOGI reject */
+ if (fc_frame_payload_op(fp) == ELS_LS_RJT)
+ qedf->flogi_failed++;
+
+ /* Complete flogi_compl so we can proceed to sending ADISCs */
+ complete(&qedf->flogi_compl);
+
+skip_stat:
+ /* Report response to libfc */
+ fc_lport_flogi_resp(seq, fp, lport);
+}
+
+static struct fc_seq *qedf_elsct_send(struct fc_lport *lport, u32 did,
+ struct fc_frame *fp, unsigned int op,
+ void (*resp)(struct fc_seq *,
+ struct fc_frame *,
+ void *),
+ void *arg, u32 timeout)
+{
+ struct qedf_ctx *qedf = lport_priv(lport);
+
+ /*
+ * Intercept FLOGI for statistic purposes. Note we use the resp
+ * callback to tell if this is really a flogi.
+ */
+ if (resp == fc_lport_flogi_resp) {
+ qedf->flogi_cnt++;
+ return fc_elsct_send(lport, did, fp, op, qedf_flogi_resp,
+ arg, timeout);
+ }
+
+ return fc_elsct_send(lport, did, fp, op, resp, arg, timeout);
+}
+
+int qedf_send_flogi(struct qedf_ctx *qedf)
+{
+ struct fc_lport *lport;
+ struct fc_frame *fp;
+
+ lport = qedf->lport;
+
+ if (!lport->tt.elsct_send)
+ return -EINVAL;
+
+ fp = fc_frame_alloc(lport, sizeof(struct fc_els_flogi));
+ if (!fp) {
+ QEDF_ERR(&(qedf->dbg_ctx), "fc_frame_alloc failed.\n");
+ return -ENOMEM;
+ }
+
+ QEDF_INFO(&(qedf->dbg_ctx), QEDF_LOG_ELS,
+ "Sending FLOGI to reestablish session with switch.\n");
+ lport->tt.elsct_send(lport, FC_FID_FLOGI, fp,
+ ELS_FLOGI, qedf_flogi_resp, lport, lport->r_a_tov);
+
+ init_completion(&qedf->flogi_compl);
+
+ return 0;
+}
+
+struct qedf_tmp_rdata_item {
+ struct fc_rport_priv *rdata;
+ struct list_head list;
+};
+
+/*
+ * This function is called if link_down_tmo is in use. If we get a link up and
+ * link_down_tmo has not expired then use just FLOGI/ADISC to recover our
+ * sessions with targets. Otherwise, just call fcoe_ctlr_link_up().
+ */
+static void qedf_link_recovery(struct work_struct *work)
+{
+ struct qedf_ctx *qedf =
+ container_of(work, struct qedf_ctx, link_recovery.work);
+ struct qedf_rport *fcport;
+ struct fc_rport_priv *rdata;
+ struct qedf_tmp_rdata_item *rdata_item, *tmp_rdata_item;
+ bool rc;
+ int retries = 30;
+ int rval, i;
+ struct list_head rdata_login_list;
+
+ INIT_LIST_HEAD(&rdata_login_list);
+
+ QEDF_INFO(&(qedf->dbg_ctx), QEDF_LOG_DISC,
+ "Link down tmo did not expire.\n");
+
+ /*
+ * Essentially reset the fcoe_ctlr here without affecting the state
+ * of the libfc structs.
+ */
+ qedf->ctlr.state = FIP_ST_LINK_WAIT;
+ fcoe_ctlr_link_down(&qedf->ctlr);
+
+ /*
+ * Bring the link up before we send the fipvlan request so libfcoe
+ * can select a new fcf in parallel
+ */
+ fcoe_ctlr_link_up(&qedf->ctlr);
+
+ /* Since the link when down and up to verify which vlan we're on */
+ qedf->fipvlan_retries = qedf_fipvlan_retries;
+ rc = qedf_initiate_fipvlan_req(qedf);
+ if (!rc)
+ return;
+
+ /*
+ * We need to wait for an FCF to be selected due to the
+ * fcoe_ctlr_link_up other the FLOGI will be rejected.
+ */
+ while (retries > 0) {
+ if (qedf->ctlr.sel_fcf) {
+ QEDF_INFO(&(qedf->dbg_ctx), QEDF_LOG_DISC,
+ "FCF reselected, proceeding with FLOGI.\n");
+ break;
+ }
+ msleep(500);
+ retries--;
+ }
+
+ if (retries < 1) {
+ QEDF_ERR(&(qedf->dbg_ctx), "Exhausted retries waiting for "
+ "FCF selection.\n");
+ return;
+ }
+
+ rval = qedf_send_flogi(qedf);
+ if (rval)
+ return;
+
+ /* Wait for FLOGI completion before proceeding with sending ADISCs */
+ i = wait_for_completion_timeout(&qedf->flogi_compl,
+ qedf->lport->r_a_tov);
+ if (i == 0) {
+ QEDF_ERR(&(qedf->dbg_ctx), "FLOGI timed out.\n");
+ return;
+ }
+
+ /*
+ * Call lport->tt.rport_login which will cause libfc to send an
+ * ADISC since the rport is in state ready.
+ */
+ rcu_read_lock();
+ list_for_each_entry_rcu(fcport, &qedf->fcports, peers) {
+ rdata = fcport->rdata;
+ if (rdata == NULL)
+ continue;
+ rdata_item = kzalloc(sizeof(struct qedf_tmp_rdata_item),
+ GFP_ATOMIC);
+ if (!rdata_item)
+ continue;
+ if (kref_get_unless_zero(&rdata->kref)) {
+ rdata_item->rdata = rdata;
+ list_add(&rdata_item->list, &rdata_login_list);
+ } else
+ kfree(rdata_item);
+ }
+ rcu_read_unlock();
+ /*
+ * Do the fc_rport_login outside of the rcu lock so we don't take a
+ * mutex in an atomic context.
+ */
+ list_for_each_entry_safe(rdata_item, tmp_rdata_item, &rdata_login_list,
+ list) {
+ list_del(&rdata_item->list);
+ fc_rport_login(rdata_item->rdata);
+ kref_put(&rdata_item->rdata->kref, fc_rport_destroy);
+ kfree(rdata_item);
+ }
+}
+
+static void qedf_update_link_speed(struct qedf_ctx *qedf,
+ struct qed_link_output *link)
+{
+ struct fc_lport *lport = qedf->lport;
+
+ lport->link_speed = FC_PORTSPEED_UNKNOWN;
+ lport->link_supported_speeds = FC_PORTSPEED_UNKNOWN;
+
+ /* Set fc_host link speed */
+ switch (link->speed) {
+ case 10000:
+ lport->link_speed = FC_PORTSPEED_10GBIT;
+ break;
+ case 25000:
+ lport->link_speed = FC_PORTSPEED_25GBIT;
+ break;
+ case 40000:
+ lport->link_speed = FC_PORTSPEED_40GBIT;
+ break;
+ case 50000:
+ lport->link_speed = FC_PORTSPEED_50GBIT;
+ break;
+ case 100000:
+ lport->link_speed = FC_PORTSPEED_100GBIT;
+ break;
+ default:
+ lport->link_speed = FC_PORTSPEED_UNKNOWN;
+ break;
+ }
+
+ /*
+ * Set supported link speed by querying the supported
+ * capabilities of the link.
+ */
+ if (link->supported_caps & SUPPORTED_10000baseKR_Full)
+ lport->link_supported_speeds |= FC_PORTSPEED_10GBIT;
+ if (link->supported_caps & SUPPORTED_25000baseKR_Full)
+ lport->link_supported_speeds |= FC_PORTSPEED_25GBIT;
+ if (link->supported_caps & SUPPORTED_40000baseLR4_Full)
+ lport->link_supported_speeds |= FC_PORTSPEED_40GBIT;
+ if (link->supported_caps & SUPPORTED_50000baseKR2_Full)
+ lport->link_supported_speeds |= FC_PORTSPEED_50GBIT;
+ if (link->supported_caps & SUPPORTED_100000baseKR4_Full)
+ lport->link_supported_speeds |= FC_PORTSPEED_100GBIT;
+ fc_host_supported_speeds(lport->host) = lport->link_supported_speeds;
+}
+
+static void qedf_link_update(void *dev, struct qed_link_output *link)
+{
+ struct qedf_ctx *qedf = (struct qedf_ctx *)dev;
+
+ if (link->link_up) {
+ QEDF_ERR(&(qedf->dbg_ctx), "LINK UP (%d GB/s).\n",
+ link->speed / 1000);
+
+ /* Cancel any pending link down work */
+ cancel_delayed_work(&qedf->link_update);
+
+ atomic_set(&qedf->link_state, QEDF_LINK_UP);
+ qedf_update_link_speed(qedf, link);
+
+ if (atomic_read(&qedf->dcbx) == QEDF_DCBX_DONE) {
+ QEDF_ERR(&(qedf->dbg_ctx), "DCBx done.\n");
+ if (atomic_read(&qedf->link_down_tmo_valid) > 0)
+ queue_delayed_work(qedf->link_update_wq,
+ &qedf->link_recovery, 0);
+ else
+ queue_delayed_work(qedf->link_update_wq,
+ &qedf->link_update, 0);
+ atomic_set(&qedf->link_down_tmo_valid, 0);
+ }
+
+ } else {
+ QEDF_ERR(&(qedf->dbg_ctx), "LINK DOWN.\n");
+
+ atomic_set(&qedf->link_state, QEDF_LINK_DOWN);
+ atomic_set(&qedf->dcbx, QEDF_DCBX_PENDING);
+ /*
+ * Flag that we're waiting for the link to come back up before
+ * informing the fcoe layer of the event.
+ */
+ if (qedf_link_down_tmo > 0) {
+ QEDF_INFO(&(qedf->dbg_ctx), QEDF_LOG_DISC,
+ "Starting link down tmo.\n");
+ atomic_set(&qedf->link_down_tmo_valid, 1);
+ }
+ qedf->vlan_id = 0;
+ qedf_update_link_speed(qedf, link);
+ queue_delayed_work(qedf->link_update_wq, &qedf->link_update,
+ qedf_link_down_tmo * HZ);
+ }
+}
+
+
+static void qedf_dcbx_handler(void *dev, struct qed_dcbx_get *get, u32 mib_type)
+{
+ struct qedf_ctx *qedf = (struct qedf_ctx *)dev;
+
+ QEDF_ERR(&(qedf->dbg_ctx), "DCBx event valid=%d enabled=%d fcoe "
+ "prio=%d.\n", get->operational.valid, get->operational.enabled,
+ get->operational.app_prio.fcoe);
+
+ if (get->operational.enabled && get->operational.valid) {
+ /* If DCBX was already negotiated on link up then just exit */
+ if (atomic_read(&qedf->dcbx) == QEDF_DCBX_DONE) {
+ QEDF_INFO(&(qedf->dbg_ctx), QEDF_LOG_DISC,
+ "DCBX already set on link up.\n");
+ return;
+ }
+
+ atomic_set(&qedf->dcbx, QEDF_DCBX_DONE);
+
+ if (atomic_read(&qedf->link_state) == QEDF_LINK_UP) {
+ if (atomic_read(&qedf->link_down_tmo_valid) > 0)
+ queue_delayed_work(qedf->link_update_wq,
+ &qedf->link_recovery, 0);
+ else
+ queue_delayed_work(qedf->link_update_wq,
+ &qedf->link_update, 0);
+ atomic_set(&qedf->link_down_tmo_valid, 0);
+ }
+ }
+
+}
+
+static u32 qedf_get_login_failures(void *cookie)
+{
+ struct qedf_ctx *qedf;
+
+ qedf = (struct qedf_ctx *)cookie;
+ return qedf->flogi_failed;
+}
+
+static struct qed_fcoe_cb_ops qedf_cb_ops = {
+ {
+ .link_update = qedf_link_update,
+ .dcbx_aen = qedf_dcbx_handler,
+ }
+};
+
+/*
+ * Various transport templates.
+ */
+
+static struct scsi_transport_template *qedf_fc_transport_template;
+static struct scsi_transport_template *qedf_fc_vport_transport_template;
+
+/*
+ * SCSI EH handlers
+ */
+static int qedf_eh_abort(struct scsi_cmnd *sc_cmd)
+{
+ struct fc_rport *rport = starget_to_rport(scsi_target(sc_cmd->device));
+ struct fc_rport_libfc_priv *rp = rport->dd_data;
+ struct qedf_rport *fcport;
+ struct fc_lport *lport;
+ struct qedf_ctx *qedf;
+ struct qedf_ioreq *io_req;
+ int rc = FAILED;
+ int rval;
+
+ if (fc_remote_port_chkready(rport)) {
+ QEDF_ERR(NULL, "rport not ready\n");
+ goto out;
+ }
+
+ lport = shost_priv(sc_cmd->device->host);
+ qedf = (struct qedf_ctx *)lport_priv(lport);
+
+ if ((lport->state != LPORT_ST_READY) || !(lport->link_up)) {
+ QEDF_ERR(&(qedf->dbg_ctx), "link not ready.\n");
+ goto out;
+ }
+
+ fcport = (struct qedf_rport *)&rp[1];
+
+ io_req = (struct qedf_ioreq *)sc_cmd->SCp.ptr;
+ if (!io_req) {
+ QEDF_ERR(&(qedf->dbg_ctx), "io_req is NULL.\n");
+ rc = SUCCESS;
+ goto out;
+ }
+
+ if (!test_bit(QEDF_CMD_OUTSTANDING, &io_req->flags) ||
+ test_bit(QEDF_CMD_IN_CLEANUP, &io_req->flags) ||
+ test_bit(QEDF_CMD_IN_ABORT, &io_req->flags)) {
+ QEDF_ERR(&(qedf->dbg_ctx), "io_req xid=0x%x already in "
+ "cleanup or abort processing or already "
+ "completed.\n", io_req->xid);
+ rc = SUCCESS;
+ goto out;
+ }
+
+ QEDF_ERR(&(qedf->dbg_ctx), "Aborting io_req sc_cmd=%p xid=0x%x "
+ "fp_idx=%d.\n", sc_cmd, io_req->xid, io_req->fp_idx);
+
+ if (qedf->stop_io_on_error) {
+ qedf_stop_all_io(qedf);
+ rc = SUCCESS;
+ goto out;
+ }
+
+ init_completion(&io_req->abts_done);
+ rval = qedf_initiate_abts(io_req, true);
+ if (rval) {
+ QEDF_ERR(&(qedf->dbg_ctx), "Failed to queue ABTS.\n");
+ goto out;
+ }
+
+ wait_for_completion(&io_req->abts_done);
+
+ if (io_req->event == QEDF_IOREQ_EV_ABORT_SUCCESS ||
+ io_req->event == QEDF_IOREQ_EV_ABORT_FAILED ||
+ io_req->event == QEDF_IOREQ_EV_CLEANUP_SUCCESS) {
+ /*
+ * If we get a reponse to the abort this is success from
+ * the perspective that all references to the command have
+ * been removed from the driver and firmware
+ */
+ rc = SUCCESS;
+ } else {
+ /* If the abort and cleanup failed then return a failure */
+ rc = FAILED;
+ }
+
+ if (rc == SUCCESS)
+ QEDF_ERR(&(qedf->dbg_ctx), "ABTS succeeded, xid=0x%x.\n",
+ io_req->xid);
+ else
+ QEDF_ERR(&(qedf->dbg_ctx), "ABTS failed, xid=0x%x.\n",
+ io_req->xid);
+
+out:
+ return rc;
+}
+
+static int qedf_eh_target_reset(struct scsi_cmnd *sc_cmd)
+{
+ QEDF_ERR(NULL, "TARGET RESET Issued...");
+ return qedf_initiate_tmf(sc_cmd, FCP_TMF_TGT_RESET);
+}
+
+static int qedf_eh_device_reset(struct scsi_cmnd *sc_cmd)
+{
+ QEDF_ERR(NULL, "LUN RESET Issued...\n");
+ return qedf_initiate_tmf(sc_cmd, FCP_TMF_LUN_RESET);
+}
+
+void qedf_wait_for_upload(struct qedf_ctx *qedf)
+{
+ while (1) {
+ if (atomic_read(&qedf->num_offloads))
+ QEDF_INFO(&(qedf->dbg_ctx), QEDF_LOG_DISC,
+ "Waiting for all uploads to complete.\n");
+ else
+ break;
+ msleep(500);
+ }
+}
+
+/* Reset the host by gracefully logging out and then logging back in */
+static int qedf_eh_host_reset(struct scsi_cmnd *sc_cmd)
+{
+ struct fc_lport *lport;
+ struct qedf_ctx *qedf;
+
+ lport = shost_priv(sc_cmd->device->host);
+
+ if (lport->vport) {
+ QEDF_ERR(NULL, "Cannot issue host reset on NPIV port.\n");
+ return SUCCESS;
+ }
+
+ qedf = (struct qedf_ctx *)lport_priv(lport);
+
+ if (atomic_read(&qedf->link_state) == QEDF_LINK_DOWN ||
+ test_bit(QEDF_UNLOADING, &qedf->flags) ||
+ test_bit(QEDF_DBG_STOP_IO, &qedf->flags))
+ return FAILED;
+
+ QEDF_ERR(&(qedf->dbg_ctx), "HOST RESET Issued...");
+
+ /* For host reset, essentially do a soft link up/down */
+ atomic_set(&qedf->link_state, QEDF_LINK_DOWN);
+ atomic_set(&qedf->dcbx, QEDF_DCBX_PENDING);
+ queue_delayed_work(qedf->link_update_wq, &qedf->link_update,
+ 0);
+ qedf_wait_for_upload(qedf);
+ atomic_set(&qedf->link_state, QEDF_LINK_UP);
+ qedf->vlan_id = 0;
+ queue_delayed_work(qedf->link_update_wq, &qedf->link_update,
+ 0);
+
+ return SUCCESS;
+}
+
+static int qedf_slave_configure(struct scsi_device *sdev)
+{
+ if (qedf_queue_depth) {
+ scsi_change_queue_depth(sdev, qedf_queue_depth);
+ }
+
+ return 0;
+}
+
+static struct scsi_host_template qedf_host_template = {
+ .module = THIS_MODULE,
+ .name = QEDF_MODULE_NAME,
+ .this_id = -1,
+ .cmd_per_lun = 3,
+ .use_clustering = ENABLE_CLUSTERING,
+ .max_sectors = 0xffff,
+ .queuecommand = qedf_queuecommand,
+ .shost_attrs = qedf_host_attrs,
+ .eh_abort_handler = qedf_eh_abort,
+ .eh_device_reset_handler = qedf_eh_device_reset, /* lun reset */
+ .eh_target_reset_handler = qedf_eh_target_reset, /* target reset */
+ .eh_host_reset_handler = qedf_eh_host_reset,
+ .slave_configure = qedf_slave_configure,
+ .dma_boundary = QED_HW_DMA_BOUNDARY,
+ .sg_tablesize = QEDF_MAX_BDS_PER_CMD,
+ .can_queue = FCOE_PARAMS_NUM_TASKS,
+};
+
+static int qedf_get_paged_crc_eof(struct sk_buff *skb, int tlen)
+{
+ int rc;
+
+ spin_lock(&qedf_global_lock);
+ rc = fcoe_get_paged_crc_eof(skb, tlen, &qedf_global);
+ spin_unlock(&qedf_global_lock);
+
+ return rc;
+}
+
+static struct qedf_rport *qedf_fcport_lookup(struct qedf_ctx *qedf, u32 port_id)
+{
+ struct qedf_rport *fcport;
+ struct fc_rport_priv *rdata;
+
+ rcu_read_lock();
+ list_for_each_entry_rcu(fcport, &qedf->fcports, peers) {
+ rdata = fcport->rdata;
+ if (rdata == NULL)
+ continue;
+ if (rdata->ids.port_id == port_id) {
+ rcu_read_unlock();
+ return fcport;
+ }
+ }
+ rcu_read_unlock();
+
+ /* Return NULL to caller to let them know fcport was not found */
+ return NULL;
+}
+
+/* Transmits an ELS frame over an offloaded session */
+static int qedf_xmit_l2_frame(struct qedf_rport *fcport, struct fc_frame *fp)
+{
+ struct fc_frame_header *fh;
+ int rc = 0;
+
+ fh = fc_frame_header_get(fp);
+ if ((fh->fh_type == FC_TYPE_ELS) &&
+ (fh->fh_r_ctl == FC_RCTL_ELS_REQ)) {
+ switch (fc_frame_payload_op(fp)) {
+ case ELS_ADISC:
+ qedf_send_adisc(fcport, fp);
+ rc = 1;
+ break;
+ }
+ }
+
+ return rc;
+}
+
+/**
+ * qedf_xmit - qedf FCoE frame transmit function
+ *
+ */
+static int qedf_xmit(struct fc_lport *lport, struct fc_frame *fp)
+{
+ struct fc_lport *base_lport;
+ struct qedf_ctx *qedf;
+ struct ethhdr *eh;
+ struct fcoe_crc_eof *cp;
+ struct sk_buff *skb;
+ struct fc_frame_header *fh;
+ struct fcoe_hdr *hp;
+ u8 sof, eof;
+ u32 crc;
+ unsigned int hlen, tlen, elen;
+ int wlen;
+ struct fc_stats *stats;
+ struct fc_lport *tmp_lport;
+ struct fc_lport *vn_port = NULL;
+ struct qedf_rport *fcport;
+ int rc;
+ u16 vlan_tci = 0;
+
+ qedf = (struct qedf_ctx *)lport_priv(lport);
+
+ fh = fc_frame_header_get(fp);
+ skb = fp_skb(fp);
+
+ /* Filter out traffic to other NPIV ports on the same host */
+ if (lport->vport)
+ base_lport = shost_priv(vport_to_shost(lport->vport));
+ else
+ base_lport = lport;
+
+ /* Flag if the destination is the base port */
+ if (base_lport->port_id == ntoh24(fh->fh_d_id)) {
+ vn_port = base_lport;
+ } else {
+ /* Got through the list of vports attached to the base_lport
+ * and see if we have a match with the destination address.
+ */
+ list_for_each_entry(tmp_lport, &base_lport->vports, list) {
+ if (tmp_lport->port_id == ntoh24(fh->fh_d_id)) {
+ vn_port = tmp_lport;
+ break;
+ }
+ }
+ }
+ if (vn_port && ntoh24(fh->fh_d_id) != FC_FID_FLOGI) {
+ struct fc_rport_priv *rdata = NULL;
+
+ QEDF_INFO(&(qedf->dbg_ctx), QEDF_LOG_LL2,
+ "Dropping FCoE frame to %06x.\n", ntoh24(fh->fh_d_id));
+ kfree_skb(skb);
+ rdata = fc_rport_lookup(lport, ntoh24(fh->fh_d_id));
+ if (rdata)
+ rdata->retries = lport->max_rport_retry_count;
+ return -EINVAL;
+ }
+ /* End NPIV filtering */
+
+ if (!qedf->ctlr.sel_fcf) {
+ kfree_skb(skb);
+ return 0;
+ }
+
+ if (!test_bit(QEDF_LL2_STARTED, &qedf->flags)) {
+ QEDF_WARN(&(qedf->dbg_ctx), "LL2 not started\n");
+ kfree_skb(skb);
+ return 0;
+ }
+
+ if (atomic_read(&qedf->link_state) != QEDF_LINK_UP) {
+ QEDF_WARN(&(qedf->dbg_ctx), "qedf link down\n");
+ kfree_skb(skb);
+ return 0;
+ }
+
+ if (unlikely(fh->fh_r_ctl == FC_RCTL_ELS_REQ)) {
+ if (fcoe_ctlr_els_send(&qedf->ctlr, lport, skb))
+ return 0;
+ }
+
+ /* Check to see if this needs to be sent on an offloaded session */
+ fcport = qedf_fcport_lookup(qedf, ntoh24(fh->fh_d_id));
+
+ if (fcport && test_bit(QEDF_RPORT_SESSION_READY, &fcport->flags)) {
+ rc = qedf_xmit_l2_frame(fcport, fp);
+ /*
+ * If the frame was successfully sent over the middle path
+ * then do not try to also send it over the LL2 path
+ */
+ if (rc)
+ return 0;
+ }
+
+ sof = fr_sof(fp);
+ eof = fr_eof(fp);
+
+ elen = sizeof(struct ethhdr);
+ hlen = sizeof(struct fcoe_hdr);
+ tlen = sizeof(struct fcoe_crc_eof);
+ wlen = (skb->len - tlen + sizeof(crc)) / FCOE_WORD_TO_BYTE;
+
+ skb->ip_summed = CHECKSUM_NONE;
+ crc = fcoe_fc_crc(fp);
+
+ /* copy port crc and eof to the skb buff */
+ if (skb_is_nonlinear(skb)) {
+ skb_frag_t *frag;
+
+ if (qedf_get_paged_crc_eof(skb, tlen)) {
+ kfree_skb(skb);
+ return -ENOMEM;
+ }
+ frag = &skb_shinfo(skb)->frags[skb_shinfo(skb)->nr_frags - 1];
+ cp = kmap_atomic(skb_frag_page(frag)) + frag->page_offset;
+ } else {
+ cp = (struct fcoe_crc_eof *)skb_put(skb, tlen);
+ }
+
+ memset(cp, 0, sizeof(*cp));
+ cp->fcoe_eof = eof;
+ cp->fcoe_crc32 = cpu_to_le32(~crc);
+ if (skb_is_nonlinear(skb)) {
+ kunmap_atomic(cp);
+ cp = NULL;
+ }
+
+
+ /* adjust skb network/transport offsets to match mac/fcoe/port */
+ skb_push(skb, elen + hlen);
+ skb_reset_mac_header(skb);
+ skb_reset_network_header(skb);
+ skb->mac_len = elen;
+ skb->protocol = htons(ETH_P_FCOE);
+
+ __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), qedf->vlan_id);
+
+ /* fill up mac and fcoe headers */
+ eh = eth_hdr(skb);
+ eh->h_proto = htons(ETH_P_FCOE);
+ if (qedf->ctlr.map_dest)
+ fc_fcoe_set_mac(eh->h_dest, fh->fh_d_id);
+ else
+ /* insert GW address */
+ ether_addr_copy(eh->h_dest, qedf->ctlr.dest_addr);
+
+ /* Set the source MAC address */
+ fc_fcoe_set_mac(eh->h_source, fh->fh_s_id);
+
+ hp = (struct fcoe_hdr *)(eh + 1);
+ memset(hp, 0, sizeof(*hp));
+ if (FC_FCOE_VER)
+ FC_FCOE_ENCAPS_VER(hp, FC_FCOE_VER);
+ hp->fcoe_sof = sof;
+
+ /*update tx stats */
+ stats = per_cpu_ptr(lport->stats, get_cpu());
+ stats->TxFrames++;
+ stats->TxWords += wlen;
+ put_cpu();
+
+ /* Get VLAN ID from skb for printing purposes */
+ __vlan_hwaccel_get_tag(skb, &vlan_tci);
+
+ /* send down to lld */
+ fr_dev(fp) = lport;
+ QEDF_INFO(&(qedf->dbg_ctx), QEDF_LOG_LL2, "FCoE frame send: "
+ "src=%06x dest=%06x r_ctl=%x type=%x vlan=%04x.\n",
+ ntoh24(fh->fh_s_id), ntoh24(fh->fh_d_id), fh->fh_r_ctl, fh->fh_type,
+ vlan_tci);
+ if (qedf_dump_frames)
+ print_hex_dump(KERN_WARNING, "fcoe: ", DUMP_PREFIX_OFFSET, 16,
+ 1, skb->data, skb->len, false);
+ qed_ops->ll2->start_xmit(qedf->cdev, skb);
+
+ return 0;
+}
+
+static int qedf_alloc_sq(struct qedf_ctx *qedf, struct qedf_rport *fcport)
+{
+ int rval = 0;
+ u32 *pbl;
+ dma_addr_t page;
+ int num_pages;
+
+ /* Calculate appropriate queue and PBL sizes */
+ fcport->sq_mem_size = SQ_NUM_ENTRIES * sizeof(struct fcoe_wqe);
+ fcport->sq_mem_size = ALIGN(fcport->sq_mem_size, QEDF_PAGE_SIZE);
+ fcport->sq_pbl_size = (fcport->sq_mem_size / QEDF_PAGE_SIZE) *
+ sizeof(void *);
+ fcport->sq_pbl_size = fcport->sq_pbl_size + QEDF_PAGE_SIZE;
+
+ fcport->sq = dma_alloc_coherent(&qedf->pdev->dev, fcport->sq_mem_size,
+ &fcport->sq_dma, GFP_KERNEL);
+ if (!fcport->sq) {
+ QEDF_WARN(&(qedf->dbg_ctx), "Could not allocate send "
+ "queue.\n");
+ rval = 1;
+ goto out;
+ }
+ memset(fcport->sq, 0, fcport->sq_mem_size);
+
+ fcport->sq_pbl = dma_alloc_coherent(&qedf->pdev->dev,
+ fcport->sq_pbl_size, &fcport->sq_pbl_dma, GFP_KERNEL);
+ if (!fcport->sq_pbl) {
+ QEDF_WARN(&(qedf->dbg_ctx), "Could not allocate send "
+ "queue PBL.\n");
+ rval = 1;
+ goto out_free_sq;
+ }
+ memset(fcport->sq_pbl, 0, fcport->sq_pbl_size);
+
+ /* Create PBL */
+ num_pages = fcport->sq_mem_size / QEDF_PAGE_SIZE;
+ page = fcport->sq_dma;
+ pbl = (u32 *)fcport->sq_pbl;
+
+ while (num_pages--) {
+ *pbl = U64_LO(page);
+ pbl++;
+ *pbl = U64_HI(page);
+ pbl++;
+ page += QEDF_PAGE_SIZE;
+ }
+
+ return rval;
+
+out_free_sq:
+ dma_free_coherent(&qedf->pdev->dev, fcport->sq_mem_size, fcport->sq,
+ fcport->sq_dma);
+out:
+ return rval;
+}
+
+static void qedf_free_sq(struct qedf_ctx *qedf, struct qedf_rport *fcport)
+{
+ if (fcport->sq_pbl)
+ dma_free_coherent(&qedf->pdev->dev, fcport->sq_pbl_size,
+ fcport->sq_pbl, fcport->sq_pbl_dma);
+ if (fcport->sq)
+ dma_free_coherent(&qedf->pdev->dev, fcport->sq_mem_size,
+ fcport->sq, fcport->sq_dma);
+}
+
+static int qedf_offload_connection(struct qedf_ctx *qedf,
+ struct qedf_rport *fcport)
+{
+ struct qed_fcoe_params_offload conn_info;
+ u32 port_id;
+ u8 lport_src_id[3];
+ int rval;
+ uint16_t total_sqe = (fcport->sq_mem_size / sizeof(struct fcoe_wqe));
+
+ QEDF_INFO(&(qedf->dbg_ctx), QEDF_LOG_CONN, "Offloading connection "
+ "portid=%06x.\n", fcport->rdata->ids.port_id);
+ rval = qed_ops->acquire_conn(qedf->cdev, &fcport->handle,
+ &fcport->fw_cid, &fcport->p_doorbell);
+ if (rval) {
+ QEDF_WARN(&(qedf->dbg_ctx), "Could not acquire connection "
+ "for portid=%06x.\n", fcport->rdata->ids.port_id);
+ rval = 1; /* For some reason qed returns 0 on failure here */
+ goto out;
+ }
+
+ QEDF_INFO(&(qedf->dbg_ctx), QEDF_LOG_CONN, "portid=%06x "
+ "fw_cid=%08x handle=%d.\n", fcport->rdata->ids.port_id,
+ fcport->fw_cid, fcport->handle);
+
+ memset(&conn_info, 0, sizeof(struct qed_fcoe_params_offload));
+
+ /* Fill in the offload connection info */
+ conn_info.sq_pbl_addr = fcport->sq_pbl_dma;
+
+ conn_info.sq_curr_page_addr = (dma_addr_t)(*(u64 *)fcport->sq_pbl);
+ conn_info.sq_next_page_addr =
+ (dma_addr_t)(*(u64 *)(fcport->sq_pbl + 8));
+
+ /* Need to use our FCoE MAC for the offload session */
+ port_id = fc_host_port_id(qedf->lport->host);
+ lport_src_id[2] = (port_id & 0x000000FF);
+ lport_src_id[1] = (port_id & 0x0000FF00) >> 8;
+ lport_src_id[0] = (port_id & 0x00FF0000) >> 16;
+ fc_fcoe_set_mac(conn_info.src_mac, lport_src_id);
+
+ ether_addr_copy(conn_info.dst_mac, qedf->ctlr.dest_addr);
+
+ conn_info.tx_max_fc_pay_len = fcport->rdata->maxframe_size;
+ conn_info.e_d_tov_timer_val = qedf->lport->e_d_tov / 20;
+ conn_info.rec_tov_timer_val = 3; /* I think this is what E3 was */
+ conn_info.rx_max_fc_pay_len = fcport->rdata->maxframe_size;
+
+ /* Set VLAN data */
+ conn_info.vlan_tag = qedf->vlan_id <<
+ FCOE_CONN_OFFLOAD_RAMROD_DATA_VLAN_ID_SHIFT;
+ conn_info.vlan_tag |=
+ qedf_default_prio << FCOE_CONN_OFFLOAD_RAMROD_DATA_PRIORITY_SHIFT;
+ conn_info.flags |= (FCOE_CONN_OFFLOAD_RAMROD_DATA_B_VLAN_FLAG_MASK <<
+ FCOE_CONN_OFFLOAD_RAMROD_DATA_B_VLAN_FLAG_SHIFT);
+
+ /* Set host port source id */
+ port_id = fc_host_port_id(qedf->lport->host);
+ fcport->sid = port_id;
+ conn_info.s_id.addr_hi = (port_id & 0x000000FF);
+ conn_info.s_id.addr_mid = (port_id & 0x0000FF00) >> 8;
+ conn_info.s_id.addr_lo = (port_id & 0x00FF0000) >> 16;
+
+ conn_info.max_conc_seqs_c3 = fcport->rdata->max_seq;
+
+ /* Set remote port destination id */
+ port_id = fcport->rdata->rport->port_id;
+ conn_info.d_id.addr_hi = (port_id & 0x000000FF);
+ conn_info.d_id.addr_mid = (port_id & 0x0000FF00) >> 8;
+ conn_info.d_id.addr_lo = (port_id & 0x00FF0000) >> 16;
+
+ conn_info.def_q_idx = 0; /* Default index for send queue? */
+
+ /* Set FC-TAPE specific flags if needed */
+ if (fcport->dev_type == QEDF_RPORT_TYPE_TAPE) {
+ QEDF_INFO(&(qedf->dbg_ctx), QEDF_LOG_CONN,
+ "Enable CONF, REC for portid=%06x.\n",
+ fcport->rdata->ids.port_id);
+ conn_info.flags |= 1 <<
+ FCOE_CONN_OFFLOAD_RAMROD_DATA_B_CONF_REQ_SHIFT;
+ conn_info.flags |=
+ ((fcport->rdata->sp_features & FC_SP_FT_SEQC) ? 1 : 0) <<
+ FCOE_CONN_OFFLOAD_RAMROD_DATA_B_REC_VALID_SHIFT;
+ }
+
+ rval = qed_ops->offload_conn(qedf->cdev, fcport->handle, &conn_info);
+ if (rval) {
+ QEDF_WARN(&(qedf->dbg_ctx), "Could not offload connection "
+ "for portid=%06x.\n", fcport->rdata->ids.port_id);
+ goto out_free_conn;
+ } else
+ QEDF_INFO(&(qedf->dbg_ctx), QEDF_LOG_CONN, "Offload "
+ "succeeded portid=%06x total_sqe=%d.\n",
+ fcport->rdata->ids.port_id, total_sqe);
+
+ spin_lock_init(&fcport->rport_lock);
+ atomic_set(&fcport->free_sqes, total_sqe);
+ return 0;
+out_free_conn:
+ qed_ops->release_conn(qedf->cdev, fcport->handle);
+out:
+ return rval;
+}
+
+#define QEDF_TERM_BUFF_SIZE 10
+static void qedf_upload_connection(struct qedf_ctx *qedf,
+ struct qedf_rport *fcport)
+{
+ void *term_params;
+ dma_addr_t term_params_dma;
+
+ /* Term params needs to be a DMA coherent buffer as qed shared the
+ * physical DMA address with the firmware. The buffer may be used in
+ * the receive path so we may eventually have to move this.
+ */
+ term_params = dma_alloc_coherent(&qedf->pdev->dev, QEDF_TERM_BUFF_SIZE,
+ &term_params_dma, GFP_KERNEL);
+
+ QEDF_INFO(&(qedf->dbg_ctx), QEDF_LOG_CONN, "Uploading connection "
+ "port_id=%06x.\n", fcport->rdata->ids.port_id);
+
+ qed_ops->destroy_conn(qedf->cdev, fcport->handle, term_params_dma);
+ qed_ops->release_conn(qedf->cdev, fcport->handle);
+
+ dma_free_coherent(&qedf->pdev->dev, QEDF_TERM_BUFF_SIZE, term_params,
+ term_params_dma);
+}
+
+static void qedf_cleanup_fcport(struct qedf_ctx *qedf,
+ struct qedf_rport *fcport)
+{
+ QEDF_INFO(&(qedf->dbg_ctx), QEDF_LOG_CONN, "Cleaning up portid=%06x.\n",
+ fcport->rdata->ids.port_id);
+
+ /* Flush any remaining i/o's before we upload the connection */
+ qedf_flush_active_ios(fcport, -1);
+
+ if (test_and_clear_bit(QEDF_RPORT_SESSION_READY, &fcport->flags))
+ qedf_upload_connection(qedf, fcport);
+ qedf_free_sq(qedf, fcport);
+ fcport->rdata = NULL;
+ fcport->qedf = NULL;
+}
+
+/**
+ * This event_callback is called after successful completion of libfc
+ * initiated target login. qedf can proceed with initiating the session
+ * establishment.
+ */
+static void qedf_rport_event_handler(struct fc_lport *lport,
+ struct fc_rport_priv *rdata,
+ enum fc_rport_event event)
+{
+ struct qedf_ctx *qedf = lport_priv(lport);
+ struct fc_rport *rport = rdata->rport;
+ struct fc_rport_libfc_priv *rp;
+ struct qedf_rport *fcport;
+ u32 port_id;
+ int rval;
+ unsigned long flags;
+
+ QEDF_INFO(&(qedf->dbg_ctx), QEDF_LOG_DISC, "event = %d, "
+ "port_id = 0x%x\n", event, rdata->ids.port_id);
+
+ switch (event) {
+ case RPORT_EV_READY:
+ if (!rport) {
+ QEDF_WARN(&(qedf->dbg_ctx), "rport is NULL.\n");
+ break;
+ }
+
+ rp = rport->dd_data;
+ fcport = (struct qedf_rport *)&rp[1];
+ fcport->qedf = qedf;
+
+ if (atomic_read(&qedf->num_offloads) >= QEDF_MAX_SESSIONS) {
+ QEDF_ERR(&(qedf->dbg_ctx), "Not offloading "
+ "portid=0x%x as max number of offloaded sessions "
+ "reached.\n", rdata->ids.port_id);
+ return;
+ }
+
+ /*
+ * Don't try to offload the session again. Can happen when we
+ * get an ADISC
+ */
+ if (test_bit(QEDF_RPORT_SESSION_READY, &fcport->flags)) {
+ QEDF_WARN(&(qedf->dbg_ctx), "Session already "
+ "offloaded, portid=0x%x.\n",
+ rdata->ids.port_id);
+ return;
+ }
+
+ if (rport->port_id == FC_FID_DIR_SERV) {
+ /*
+ * qedf_rport structure doesn't exist for
+ * directory server.
+ * We should not come here, as lport will
+ * take care of fabric login
+ */
+ QEDF_WARN(&(qedf->dbg_ctx), "rport struct does not "
+ "exist for dir server port_id=%x\n",
+ rdata->ids.port_id);
+ break;
+ }
+
+ if (rdata->spp_type != FC_TYPE_FCP) {
+ QEDF_INFO(&(qedf->dbg_ctx), QEDF_LOG_DISC,
+ "Not offlading since since spp type isn't FCP\n");
+ break;
+ }
+ if (!(rdata->ids.roles & FC_RPORT_ROLE_FCP_TARGET)) {
+ QEDF_INFO(&(qedf->dbg_ctx), QEDF_LOG_DISC,
+ "Not FCP target so not offloading\n");
+ break;
+ }
+
+ fcport->rdata = rdata;
+ fcport->rport = rport;
+
+ rval = qedf_alloc_sq(qedf, fcport);
+ if (rval) {
+ qedf_cleanup_fcport(qedf, fcport);
+ break;
+ }
+
+ /* Set device type */
+ if (rdata->flags & FC_RP_FLAGS_RETRY &&
+ rdata->ids.roles & FC_RPORT_ROLE_FCP_TARGET &&
+ !(rdata->ids.roles & FC_RPORT_ROLE_FCP_INITIATOR)) {
+ fcport->dev_type = QEDF_RPORT_TYPE_TAPE;
+ QEDF_INFO(&(qedf->dbg_ctx), QEDF_LOG_DISC,
+ "portid=%06x is a TAPE device.\n",
+ rdata->ids.port_id);
+ } else {
+ fcport->dev_type = QEDF_RPORT_TYPE_DISK;
+ }
+
+ rval = qedf_offload_connection(qedf, fcport);
+ if (rval) {
+ qedf_cleanup_fcport(qedf, fcport);
+ break;
+ }
+
+ /* Add fcport to list of qedf_ctx list of offloaded ports */
+ spin_lock_irqsave(&qedf->hba_lock, flags);
+ list_add_rcu(&fcport->peers, &qedf->fcports);
+ spin_unlock_irqrestore(&qedf->hba_lock, flags);
+
+ /*
+ * Set the session ready bit to let everyone know that this
+ * connection is ready for I/O
+ */
+ set_bit(QEDF_RPORT_SESSION_READY, &fcport->flags);
+ atomic_inc(&qedf->num_offloads);
+
+ break;
+ case RPORT_EV_LOGO:
+ case RPORT_EV_FAILED:
+ case RPORT_EV_STOP:
+ port_id = rdata->ids.port_id;
+ if (port_id == FC_FID_DIR_SERV)
+ break;
+
+ if (!rport) {
+ QEDF_INFO(&(qedf->dbg_ctx), QEDF_LOG_DISC,
+ "port_id=%x - rport notcreated Yet!!\n", port_id);
+ break;
+ }
+ rp = rport->dd_data;
+ /*
+ * Perform session upload. Note that rdata->peers is already
+ * removed from disc->rports list before we get this event.
+ */
+ fcport = (struct qedf_rport *)&rp[1];
+
+ /* Only free this fcport if it is offloaded already */
+ if (test_bit(QEDF_RPORT_SESSION_READY, &fcport->flags)) {
+ set_bit(QEDF_RPORT_UPLOADING_CONNECTION, &fcport->flags);
+ qedf_cleanup_fcport(qedf, fcport);
+
+ /*
+ * Remove fcport to list of qedf_ctx list of offloaded
+ * ports
+ */
+ spin_lock_irqsave(&qedf->hba_lock, flags);
+ list_del_rcu(&fcport->peers);
+ spin_unlock_irqrestore(&qedf->hba_lock, flags);
+
+ clear_bit(QEDF_RPORT_UPLOADING_CONNECTION,
+ &fcport->flags);
+ atomic_dec(&qedf->num_offloads);
+ }
+
+ break;
+
+ case RPORT_EV_NONE:
+ break;
+ }
+}
+
+static void qedf_abort_io(struct fc_lport *lport)
+{
+ /* NO-OP but need to fill in the template */
+}
+
+static void qedf_fcp_cleanup(struct fc_lport *lport)
+{
+ /*
+ * NO-OP but need to fill in template to prevent a NULL
+ * function pointer dereference during link down. I/Os
+ * will be flushed when port is uploaded.
+ */
+}
+
+static struct libfc_function_template qedf_lport_template = {
+ .frame_send = qedf_xmit,
+ .fcp_abort_io = qedf_abort_io,
+ .fcp_cleanup = qedf_fcp_cleanup,
+ .rport_event_callback = qedf_rport_event_handler,
+ .elsct_send = qedf_elsct_send,
+};
+
+static void qedf_fcoe_ctlr_setup(struct qedf_ctx *qedf)
+{
+ fcoe_ctlr_init(&qedf->ctlr, FIP_ST_AUTO);
+
+ qedf->ctlr.send = qedf_fip_send;
+ qedf->ctlr.update_mac = qedf_update_src_mac;
+ qedf->ctlr.get_src_addr = qedf_get_src_mac;
+ ether_addr_copy(qedf->ctlr.ctl_src_addr, qedf->mac);
+}
+
+static int qedf_lport_setup(struct qedf_ctx *qedf)
+{
+ struct fc_lport *lport = qedf->lport;
+
+ lport->link_up = 0;
+ lport->max_retry_count = QEDF_FLOGI_RETRY_CNT;
+ lport->max_rport_retry_count = QEDF_RPORT_RETRY_CNT;
+ lport->service_params = (FCP_SPPF_INIT_FCN | FCP_SPPF_RD_XRDY_DIS |
+ FCP_SPPF_RETRY | FCP_SPPF_CONF_COMPL);
+ lport->boot_time = jiffies;
+ lport->e_d_tov = 2 * 1000;
+ lport->r_a_tov = 10 * 1000;
+
+ /* Set NPIV support */
+ lport->does_npiv = 1;
+ fc_host_max_npiv_vports(lport->host) = QEDF_MAX_NPIV;
+
+ fc_set_wwnn(lport, qedf->wwnn);
+ fc_set_wwpn(lport, qedf->wwpn);
+
+ fcoe_libfc_config(lport, &qedf->ctlr, &qedf_lport_template, 0);
+
+ /* Allocate the exchange manager */
+ fc_exch_mgr_alloc(lport, FC_CLASS_3, qedf->max_scsi_xid + 1,
+ qedf->max_els_xid, NULL);
+
+ if (fc_lport_init_stats(lport))
+ return -ENOMEM;
+
+ /* Finish lport config */
+ fc_lport_config(lport);
+
+ /* Set max frame size */
+ fc_set_mfs(lport, QEDF_MFS);
+ fc_host_maxframe_size(lport->host) = lport->mfs;
+
+ /* Set default dev_loss_tmo based on module parameter */
+ fc_host_dev_loss_tmo(lport->host) = qedf_dev_loss_tmo;
+
+ /* Set symbolic node name */
+ snprintf(fc_host_symbolic_name(lport->host), 256,
+ "QLogic %s v%s", QEDF_MODULE_NAME, QEDF_VERSION);
+
+ return 0;
+}
+
+/*
+ * NPIV functions
+ */
+
+static int qedf_vport_libfc_config(struct fc_vport *vport,
+ struct fc_lport *lport)
+{
+ lport->link_up = 0;
+ lport->qfull = 0;
+ lport->max_retry_count = QEDF_FLOGI_RETRY_CNT;
+ lport->max_rport_retry_count = QEDF_RPORT_RETRY_CNT;
+ lport->service_params = (FCP_SPPF_INIT_FCN | FCP_SPPF_RD_XRDY_DIS |
+ FCP_SPPF_RETRY | FCP_SPPF_CONF_COMPL);
+ lport->boot_time = jiffies;
+ lport->e_d_tov = 2 * 1000;
+ lport->r_a_tov = 10 * 1000;
+ lport->does_npiv = 1; /* Temporary until we add NPIV support */
+
+ /* Allocate stats for vport */
+ if (fc_lport_init_stats(lport))
+ return -ENOMEM;
+
+ /* Finish lport config */
+ fc_lport_config(lport);
+
+ /* offload related configuration */
+ lport->crc_offload = 0;
+ lport->seq_offload = 0;
+ lport->lro_enabled = 0;
+ lport->lro_xid = 0;
+ lport->lso_max = 0;
+
+ return 0;
+}
+
+static int qedf_vport_create(struct fc_vport *vport, bool disabled)
+{
+ struct Scsi_Host *shost = vport_to_shost(vport);
+ struct fc_lport *n_port = shost_priv(shost);
+ struct fc_lport *vn_port;
+ struct qedf_ctx *base_qedf = lport_priv(n_port);
+ struct qedf_ctx *vport_qedf;
+
+ char buf[32];
+ int rc = 0;
+
+ rc = fcoe_validate_vport_create(vport);
+ if (rc) {
+ fcoe_wwn_to_str(vport->port_name, buf, sizeof(buf));
+ QEDF_WARN(&(base_qedf->dbg_ctx), "Failed to create vport, "
+ "WWPN (0x%s) already exists.\n", buf);
+ goto err1;
+ }
+
+ if (atomic_read(&base_qedf->link_state) != QEDF_LINK_UP) {
+ QEDF_WARN(&(base_qedf->dbg_ctx), "Cannot create vport "
+ "because link is not up.\n");
+ rc = -EIO;
+ goto err1;
+ }
+
+ vn_port = libfc_vport_create(vport, sizeof(struct qedf_ctx));
+ if (!vn_port) {
+ QEDF_WARN(&(base_qedf->dbg_ctx), "Could not create lport "
+ "for vport.\n");
+ rc = -ENOMEM;
+ goto err1;
+ }
+
+ fcoe_wwn_to_str(vport->port_name, buf, sizeof(buf));
+ QEDF_ERR(&(base_qedf->dbg_ctx), "Creating NPIV port, WWPN=%s.\n",
+ buf);
+
+ /* Copy some fields from base_qedf */
+ vport_qedf = lport_priv(vn_port);
+ memcpy(vport_qedf, base_qedf, sizeof(struct qedf_ctx));
+
+ /* Set qedf data specific to this vport */
+ vport_qedf->lport = vn_port;
+ /* Use same hba_lock as base_qedf */
+ vport_qedf->hba_lock = base_qedf->hba_lock;
+ vport_qedf->pdev = base_qedf->pdev;
+ vport_qedf->cmd_mgr = base_qedf->cmd_mgr;
+ init_completion(&vport_qedf->flogi_compl);
+ INIT_LIST_HEAD(&vport_qedf->fcports);
+
+ rc = qedf_vport_libfc_config(vport, vn_port);
+ if (rc) {
+ QEDF_ERR(&(base_qedf->dbg_ctx), "Could not allocate memory "
+ "for lport stats.\n");
+ goto err2;
+ }
+
+ fc_set_wwnn(vn_port, vport->node_name);
+ fc_set_wwpn(vn_port, vport->port_name);
+ vport_qedf->wwnn = vn_port->wwnn;
+ vport_qedf->wwpn = vn_port->wwpn;
+
+ vn_port->host->transportt = qedf_fc_vport_transport_template;
+ vn_port->host->can_queue = QEDF_MAX_ELS_XID;
+ vn_port->host->max_lun = qedf_max_lun;
+ vn_port->host->sg_tablesize = QEDF_MAX_BDS_PER_CMD;
+ vn_port->host->max_cmd_len = QEDF_MAX_CDB_LEN;
+
+ rc = scsi_add_host(vn_port->host, &vport->dev);
+ if (rc) {
+ QEDF_WARN(&(base_qedf->dbg_ctx), "Error adding Scsi_Host.\n");
+ goto err2;
+ }
+
+ /* Set default dev_loss_tmo based on module parameter */
+ fc_host_dev_loss_tmo(vn_port->host) = qedf_dev_loss_tmo;
+
+ /* Init libfc stuffs */
+ memcpy(&vn_port->tt, &qedf_lport_template,
+ sizeof(qedf_lport_template));
+ fc_exch_init(vn_port);
+ fc_elsct_init(vn_port);
+ fc_lport_init(vn_port);
+ fc_disc_init(vn_port);
+ fc_disc_config(vn_port, vn_port);
+
+
+ /* Allocate the exchange manager */
+ shost = vport_to_shost(vport);
+ n_port = shost_priv(shost);
+ fc_exch_mgr_list_clone(n_port, vn_port);
+
+ /* Set max frame size */
+ fc_set_mfs(vn_port, QEDF_MFS);
+
+ fc_host_port_type(vn_port->host) = FC_PORTTYPE_UNKNOWN;
+
+ if (disabled) {
+ fc_vport_set_state(vport, FC_VPORT_DISABLED);
+ } else {
+ vn_port->boot_time = jiffies;
+ fc_fabric_login(vn_port);
+ fc_vport_setlink(vn_port);
+ }
+
+ QEDF_INFO(&(base_qedf->dbg_ctx), QEDF_LOG_NPIV, "vn_port=%p.\n",
+ vn_port);
+
+ /* Set up debug context for vport */
+ vport_qedf->dbg_ctx.host_no = vn_port->host->host_no;
+ vport_qedf->dbg_ctx.pdev = base_qedf->pdev;
+
+err2:
+ scsi_host_put(vn_port->host);
+err1:
+ return rc;
+}
+
+static int qedf_vport_destroy(struct fc_vport *vport)
+{
+ struct Scsi_Host *shost = vport_to_shost(vport);
+ struct fc_lport *n_port = shost_priv(shost);
+ struct fc_lport *vn_port = vport->dd_data;
+
+ mutex_lock(&n_port->lp_mutex);
+ list_del(&vn_port->list);
+ mutex_unlock(&n_port->lp_mutex);
+
+ fc_fabric_logoff(vn_port);
+ fc_lport_destroy(vn_port);
+
+ /* Detach from scsi-ml */
+ fc_remove_host(vn_port->host);
+ scsi_remove_host(vn_port->host);
+
+ /*
+ * Only try to release the exchange manager if the vn_port
+ * configuration is complete.
+ */
+ if (vn_port->state == LPORT_ST_READY)
+ fc_exch_mgr_free(vn_port);
+
+ /* Free memory used by statistical counters */
+ fc_lport_free_stats(vn_port);
+
+ /* Release Scsi_Host */
+ if (vn_port->host)
+ scsi_host_put(vn_port->host);
+
+ return 0;
+}
+
+static int qedf_vport_disable(struct fc_vport *vport, bool disable)
+{
+ struct fc_lport *lport = vport->dd_data;
+
+ if (disable) {
+ fc_vport_set_state(vport, FC_VPORT_DISABLED);
+ fc_fabric_logoff(lport);
+ } else {
+ lport->boot_time = jiffies;
+ fc_fabric_login(lport);
+ fc_vport_setlink(lport);
+ }
+ return 0;
+}
+
+/*
+ * During removal we need to wait for all the vports associated with a port
+ * to be destroyed so we avoid a race condition where libfc is still trying
+ * to reap vports while the driver remove function has already reaped the
+ * driver contexts associated with the physical port.
+ */
+static void qedf_wait_for_vport_destroy(struct qedf_ctx *qedf)
+{
+ struct fc_host_attrs *fc_host = shost_to_fc_host(qedf->lport->host);
+
+ QEDF_INFO(&(qedf->dbg_ctx), QEDF_LOG_NPIV,
+ "Entered.\n");
+ while (fc_host->npiv_vports_inuse > 0) {
+ QEDF_INFO(&(qedf->dbg_ctx), QEDF_LOG_NPIV,
+ "Waiting for all vports to be reaped.\n");
+ msleep(1000);
+ }
+}
+
+/**
+ * qedf_fcoe_reset - Resets the fcoe
+ *
+ * @shost: shost the reset is from
+ *
+ * Returns: always 0
+ */
+static int qedf_fcoe_reset(struct Scsi_Host *shost)
+{
+ struct fc_lport *lport = shost_priv(shost);
+
+ fc_fabric_logoff(lport);
+ fc_fabric_login(lport);
+ return 0;
+}
+
+static struct fc_host_statistics *qedf_fc_get_host_stats(struct Scsi_Host
+ *shost)
+{
+ struct fc_host_statistics *qedf_stats;
+ struct fc_lport *lport = shost_priv(shost);
+ struct qedf_ctx *qedf = lport_priv(lport);
+ struct qed_fcoe_stats *fw_fcoe_stats;
+
+ qedf_stats = fc_get_host_stats(shost);
+
+ /* We don't collect offload stats for specific NPIV ports */
+ if (lport->vport)
+ goto out;
+
+ fw_fcoe_stats = kmalloc(sizeof(struct qed_fcoe_stats), GFP_KERNEL);
+ if (!fw_fcoe_stats) {
+ QEDF_ERR(&(qedf->dbg_ctx), "Could not allocate memory for "
+ "fw_fcoe_stats.\n");
+ goto out;
+ }
+
+ /* Query firmware for offload stats */
+ qed_ops->get_stats(qedf->cdev, fw_fcoe_stats);
+
+ /*
+ * The expectation is that we add our offload stats to the stats
+ * being maintained by libfc each time the fc_get_host_status callback
+ * is invoked. The additions are not carried over for each call to
+ * the fc_get_host_stats callback.
+ */
+ qedf_stats->tx_frames += fw_fcoe_stats->fcoe_tx_data_pkt_cnt +
+ fw_fcoe_stats->fcoe_tx_xfer_pkt_cnt +
+ fw_fcoe_stats->fcoe_tx_other_pkt_cnt;
+ qedf_stats->rx_frames += fw_fcoe_stats->fcoe_rx_data_pkt_cnt +
+ fw_fcoe_stats->fcoe_rx_xfer_pkt_cnt +
+ fw_fcoe_stats->fcoe_rx_other_pkt_cnt;
+ qedf_stats->fcp_input_megabytes +=
+ do_div(fw_fcoe_stats->fcoe_rx_byte_cnt, 1000000);
+ qedf_stats->fcp_output_megabytes +=
+ do_div(fw_fcoe_stats->fcoe_tx_byte_cnt, 1000000);
+ qedf_stats->rx_words += fw_fcoe_stats->fcoe_rx_byte_cnt / 4;
+ qedf_stats->tx_words += fw_fcoe_stats->fcoe_tx_byte_cnt / 4;
+ qedf_stats->invalid_crc_count +=
+ fw_fcoe_stats->fcoe_silent_drop_pkt_crc_error_cnt;
+ qedf_stats->dumped_frames =
+ fw_fcoe_stats->fcoe_silent_drop_total_pkt_cnt;
+ qedf_stats->error_frames +=
+ fw_fcoe_stats->fcoe_silent_drop_total_pkt_cnt;
+ qedf_stats->fcp_input_requests += qedf->input_requests;
+ qedf_stats->fcp_output_requests += qedf->output_requests;
+ qedf_stats->fcp_control_requests += qedf->control_requests;
+ qedf_stats->fcp_packet_aborts += qedf->packet_aborts;
+ qedf_stats->fcp_frame_alloc_failures += qedf->alloc_failures;
+
+ kfree(fw_fcoe_stats);
+out:
+ return qedf_stats;
+}
+
+static struct fc_function_template qedf_fc_transport_fn = {
+ .show_host_node_name = 1,
+ .show_host_port_name = 1,
+ .show_host_supported_classes = 1,
+ .show_host_supported_fc4s = 1,
+ .show_host_active_fc4s = 1,
+ .show_host_maxframe_size = 1,
+
+ .show_host_port_id = 1,
+ .show_host_supported_speeds = 1,
+ .get_host_speed = fc_get_host_speed,
+ .show_host_speed = 1,
+ .show_host_port_type = 1,
+ .get_host_port_state = fc_get_host_port_state,
+ .show_host_port_state = 1,
+ .show_host_symbolic_name = 1,
+
+ /*
+ * Tell FC transport to allocate enough space to store the backpointer
+ * for the associate qedf_rport struct.
+ */
+ .dd_fcrport_size = (sizeof(struct fc_rport_libfc_priv) +
+ sizeof(struct qedf_rport)),
+ .show_rport_maxframe_size = 1,
+ .show_rport_supported_classes = 1,
+ .show_host_fabric_name = 1,
+ .show_starget_node_name = 1,
+ .show_starget_port_name = 1,
+ .show_starget_port_id = 1,
+ .set_rport_dev_loss_tmo = fc_set_rport_loss_tmo,
+ .show_rport_dev_loss_tmo = 1,
+ .get_fc_host_stats = qedf_fc_get_host_stats,
+ .issue_fc_host_lip = qedf_fcoe_reset,
+ .vport_create = qedf_vport_create,
+ .vport_delete = qedf_vport_destroy,
+ .vport_disable = qedf_vport_disable,
+ .bsg_request = fc_lport_bsg_request,
+};
+
+static struct fc_function_template qedf_fc_vport_transport_fn = {
+ .show_host_node_name = 1,
+ .show_host_port_name = 1,
+ .show_host_supported_classes = 1,
+ .show_host_supported_fc4s = 1,
+ .show_host_active_fc4s = 1,
+ .show_host_maxframe_size = 1,
+ .show_host_port_id = 1,
+ .show_host_supported_speeds = 1,
+ .get_host_speed = fc_get_host_speed,
+ .show_host_speed = 1,
+ .show_host_port_type = 1,
+ .get_host_port_state = fc_get_host_port_state,
+ .show_host_port_state = 1,
+ .show_host_symbolic_name = 1,
+ .dd_fcrport_size = (sizeof(struct fc_rport_libfc_priv) +
+ sizeof(struct qedf_rport)),
+ .show_rport_maxframe_size = 1,
+ .show_rport_supported_classes = 1,
+ .show_host_fabric_name = 1,
+ .show_starget_node_name = 1,
+ .show_starget_port_name = 1,
+ .show_starget_port_id = 1,
+ .set_rport_dev_loss_tmo = fc_set_rport_loss_tmo,
+ .show_rport_dev_loss_tmo = 1,
+ .get_fc_host_stats = fc_get_host_stats,
+ .issue_fc_host_lip = qedf_fcoe_reset,
+ .bsg_request = fc_lport_bsg_request,
+};
+
+static bool qedf_fp_has_work(struct qedf_fastpath *fp)
+{
+ struct qedf_ctx *qedf = fp->qedf;
+ struct global_queue *que;
+ struct qed_sb_info *sb_info = fp->sb_info;
+ struct status_block *sb = sb_info->sb_virt;
+ u16 prod_idx;
+
+ /* Get the pointer to the global CQ this completion is on */
+ que = qedf->global_queues[fp->sb_id];
+
+ /* Be sure all responses have been written to PI */
+ rmb();
+
+ /* Get the current firmware producer index */
+ prod_idx = sb->pi_array[QEDF_FCOE_PARAMS_GL_RQ_PI];
+
+ return (que->cq_prod_idx != prod_idx);
+}
+
+/*
+ * Interrupt handler code.
+ */
+
+/* Process completion queue and copy CQE contents for deferred processesing
+ *
+ * Return true if we should wake the I/O thread, false if not.
+ */
+static bool qedf_process_completions(struct qedf_fastpath *fp)
+{
+ struct qedf_ctx *qedf = fp->qedf;
+ struct qed_sb_info *sb_info = fp->sb_info;
+ struct status_block *sb = sb_info->sb_virt;
+ struct global_queue *que;
+ u16 prod_idx;
+ struct fcoe_cqe *cqe;
+ struct qedf_io_work *io_work;
+ int num_handled = 0;
+ unsigned int cpu;
+ struct qedf_ioreq *io_req = NULL;
+ u16 xid;
+ u16 new_cqes;
+ u32 comp_type;
+
+ /* Get the current firmware producer index */
+ prod_idx = sb->pi_array[QEDF_FCOE_PARAMS_GL_RQ_PI];
+
+ /* Get the pointer to the global CQ this completion is on */
+ que = qedf->global_queues[fp->sb_id];
+
+ /* Calculate the amount of new elements since last processing */
+ new_cqes = (prod_idx >= que->cq_prod_idx) ?
+ (prod_idx - que->cq_prod_idx) :
+ 0x10000 - que->cq_prod_idx + prod_idx;
+
+ /* Save producer index */
+ que->cq_prod_idx = prod_idx;
+
+ while (new_cqes) {
+ fp->completions++;
+ num_handled++;
+ cqe = &que->cq[que->cq_cons_idx];
+
+ comp_type = (cqe->cqe_data >> FCOE_CQE_CQE_TYPE_SHIFT) &
+ FCOE_CQE_CQE_TYPE_MASK;
+
+ /*
+ * Process unsolicited CQEs directly in the interrupt handler
+ * sine we need the fastpath ID
+ */
+ if (comp_type == FCOE_UNSOLIC_CQE_TYPE) {
+ QEDF_INFO(&(qedf->dbg_ctx), QEDF_LOG_UNSOL,
+ "Unsolicated CQE.\n");
+ qedf_process_unsol_compl(qedf, fp->sb_id, cqe);
+ /*
+ * Don't add a work list item. Increment consumer
+ * consumer index and move on.
+ */
+ goto inc_idx;
+ }
+
+ xid = cqe->cqe_data & FCOE_CQE_TASK_ID_MASK;
+ io_req = &qedf->cmd_mgr->cmds[xid];
+
+ /*
+ * Figure out which percpu thread we should queue this I/O
+ * on.
+ */
+ if (!io_req)
+ /* If there is not io_req assocated with this CQE
+ * just queue it on CPU 0
+ */
+ cpu = 0;
+ else {
+ cpu = io_req->cpu;
+ io_req->int_cpu = smp_processor_id();
+ }
+
+ io_work = mempool_alloc(qedf->io_mempool, GFP_ATOMIC);
+ if (!io_work) {
+ QEDF_WARN(&(qedf->dbg_ctx), "Could not allocate "
+ "work for I/O completion.\n");
+ continue;
+ }
+ memset(io_work, 0, sizeof(struct qedf_io_work));
+
+ INIT_WORK(&io_work->work, qedf_fp_io_handler);
+
+ /* Copy contents of CQE for deferred processing */
+ memcpy(&io_work->cqe, cqe, sizeof(struct fcoe_cqe));
+
+ io_work->qedf = fp->qedf;
+ io_work->fp = NULL; /* Only used for unsolicited frames */
+
+ queue_work_on(cpu, qedf_io_wq, &io_work->work);
+
+inc_idx:
+ que->cq_cons_idx++;
+ if (que->cq_cons_idx == fp->cq_num_entries)
+ que->cq_cons_idx = 0;
+ new_cqes--;
+ }
+
+ return true;
+}
+
+
+/* MSI-X fastpath handler code */
+static irqreturn_t qedf_msix_handler(int irq, void *dev_id)
+{
+ struct qedf_fastpath *fp = dev_id;
+
+ if (!fp) {
+ QEDF_ERR(NULL, "fp is null.\n");
+ return IRQ_HANDLED;
+ }
+ if (!fp->sb_info) {
+ QEDF_ERR(NULL, "fp->sb_info in null.");
+ return IRQ_HANDLED;
+ }
+
+ /*
+ * Disable interrupts for this status block while we process new
+ * completions
+ */
+ qed_sb_ack(fp->sb_info, IGU_INT_DISABLE, 0 /*do not update*/);
+
+ while (1) {
+ qedf_process_completions(fp);
+
+ if (qedf_fp_has_work(fp) == 0) {
+ /* Update the sb information */
+ qed_sb_update_sb_idx(fp->sb_info);
+
+ /* Check for more work */
+ rmb();
+
+ if (qedf_fp_has_work(fp) == 0) {
+ /* Re-enable interrupts */
+ qed_sb_ack(fp->sb_info, IGU_INT_ENABLE, 1);
+ return IRQ_HANDLED;
+ }
+ }
+ }
+
+ /* Do we ever want to break out of above loop? */
+ return IRQ_HANDLED;
+}
+
+/* simd handler for MSI/INTa */
+static void qedf_simd_int_handler(void *cookie)
+{
+ /* Cookie is qedf_ctx struct */
+ struct qedf_ctx *qedf = (struct qedf_ctx *)cookie;
+
+ QEDF_WARN(&(qedf->dbg_ctx), "qedf=%p.\n", qedf);
+}
+
+#define QEDF_SIMD_HANDLER_NUM 0
+static void qedf_sync_free_irqs(struct qedf_ctx *qedf)
+{
+ int i;
+
+ if (qedf->int_info.msix_cnt) {
+ for (i = 0; i < qedf->int_info.used_cnt; i++) {
+ synchronize_irq(qedf->int_info.msix[i].vector);
+ irq_set_affinity_hint(qedf->int_info.msix[i].vector,
+ NULL);
+ irq_set_affinity_notifier(qedf->int_info.msix[i].vector,
+ NULL);
+ free_irq(qedf->int_info.msix[i].vector,
+ &qedf->fp_array[i]);
+ }
+ } else
+ qed_ops->common->simd_handler_clean(qedf->cdev,
+ QEDF_SIMD_HANDLER_NUM);
+
+ qedf->int_info.used_cnt = 0;
+ qed_ops->common->set_fp_int(qedf->cdev, 0);
+}
+
+static int qedf_request_msix_irq(struct qedf_ctx *qedf)
+{
+ int i, rc, cpu;
+
+ cpu = cpumask_first(cpu_online_mask);
+ for (i = 0; i < qedf->num_queues; i++) {
+ rc = request_irq(qedf->int_info.msix[i].vector,
+ qedf_msix_handler, 0, "qedf", &qedf->fp_array[i]);
+
+ if (rc) {
+ QEDF_WARN(&(qedf->dbg_ctx), "request_irq failed.\n");
+ qedf_sync_free_irqs(qedf);
+ return rc;
+ }
+
+ qedf->int_info.used_cnt++;
+ rc = irq_set_affinity_hint(qedf->int_info.msix[i].vector,
+ get_cpu_mask(cpu));
+ cpu = cpumask_next(cpu, cpu_online_mask);
+ }
+
+ return 0;
+}
+
+static int qedf_setup_int(struct qedf_ctx *qedf)
+{
+ int rc = 0;
+
+ /*
+ * Learn interrupt configuration
+ */
+ rc = qed_ops->common->set_fp_int(qedf->cdev, num_online_cpus());
+
+ rc = qed_ops->common->get_fp_int(qedf->cdev, &qedf->int_info);
+ if (rc)
+ return 0;
+
+ QEDF_INFO(&(qedf->dbg_ctx), QEDF_LOG_DISC, "Number of msix_cnt = "
+ "0x%x num of cpus = 0x%x\n", qedf->int_info.msix_cnt,
+ num_online_cpus());
+
+ if (qedf->int_info.msix_cnt)
+ return qedf_request_msix_irq(qedf);
+
+ qed_ops->common->simd_handler_config(qedf->cdev, &qedf,
+ QEDF_SIMD_HANDLER_NUM, qedf_simd_int_handler);
+ qedf->int_info.used_cnt = 1;
+
+ return 0;
+}
+
+/* Main function for libfc frame reception */
+static void qedf_recv_frame(struct qedf_ctx *qedf,
+ struct sk_buff *skb)
+{
+ u32 fr_len;
+ struct fc_lport *lport;
+ struct fc_frame_header *fh;
+ struct fcoe_crc_eof crc_eof;
+ struct fc_frame *fp;
+ u8 *mac = NULL;
+ u8 *dest_mac = NULL;
+ struct fcoe_hdr *hp;
+ struct qedf_rport *fcport;
+
+ lport = qedf->lport;
+ if (lport == NULL || lport->state == LPORT_ST_DISABLED) {
+ QEDF_WARN(NULL, "Invalid lport struct or lport disabled.\n");
+ kfree_skb(skb);
+ return;
+ }
+
+ if (skb_is_nonlinear(skb))
+ skb_linearize(skb);
+ mac = eth_hdr(skb)->h_source;
+ dest_mac = eth_hdr(skb)->h_dest;
+
+ /* Pull the header */
+ hp = (struct fcoe_hdr *)skb->data;
+ fh = (struct fc_frame_header *) skb_transport_header(skb);
+ skb_pull(skb, sizeof(struct fcoe_hdr));
+ fr_len = skb->len - sizeof(struct fcoe_crc_eof);
+
+ fp = (struct fc_frame *)skb;
+ fc_frame_init(fp);
+ fr_dev(fp) = lport;
+ fr_sof(fp) = hp->fcoe_sof;
+ if (skb_copy_bits(skb, fr_len, &crc_eof, sizeof(crc_eof))) {
+ kfree_skb(skb);
+ return;
+ }
+ fr_eof(fp) = crc_eof.fcoe_eof;
+ fr_crc(fp) = crc_eof.fcoe_crc32;
+ if (pskb_trim(skb, fr_len)) {
+ kfree_skb(skb);
+ return;
+ }
+
+ fh = fc_frame_header_get(fp);
+
+ if (fh->fh_r_ctl == FC_RCTL_DD_SOL_DATA &&
+ fh->fh_type == FC_TYPE_FCP) {
+ /* Drop FCP data. We dont this in L2 path */
+ kfree_skb(skb);
+ return;
+ }
+ if (fh->fh_r_ctl == FC_RCTL_ELS_REQ &&
+ fh->fh_type == FC_TYPE_ELS) {
+ switch (fc_frame_payload_op(fp)) {
+ case ELS_LOGO:
+ if (ntoh24(fh->fh_s_id) == FC_FID_FLOGI) {
+ /* drop non-FIP LOGO */
+ kfree_skb(skb);
+ return;
+ }
+ break;
+ }
+ }
+
+ if (fh->fh_r_ctl == FC_RCTL_BA_ABTS) {
+ /* Drop incoming ABTS */
+ kfree_skb(skb);
+ return;
+ }
+
+ /*
+ * If a connection is uploading, drop incoming FCoE frames as there
+ * is a small window where we could try to return a frame while libfc
+ * is trying to clean things up.
+ */
+
+ /* Get fcport associated with d_id if it exists */
+ fcport = qedf_fcport_lookup(qedf, ntoh24(fh->fh_d_id));
+
+ if (fcport && test_bit(QEDF_RPORT_UPLOADING_CONNECTION,
+ &fcport->flags)) {
+ QEDF_INFO(&(qedf->dbg_ctx), QEDF_LOG_LL2,
+ "Connection uploading, dropping fp=%p.\n", fp);
+ kfree_skb(skb);
+ return;
+ }
+
+ QEDF_INFO(&(qedf->dbg_ctx), QEDF_LOG_LL2, "FCoE frame receive: "
+ "skb=%p fp=%p src=%06x dest=%06x r_ctl=%x fh_type=%x.\n", skb, fp,
+ ntoh24(fh->fh_s_id), ntoh24(fh->fh_d_id), fh->fh_r_ctl,
+ fh->fh_type);
+ if (qedf_dump_frames)
+ print_hex_dump(KERN_WARNING, "fcoe: ", DUMP_PREFIX_OFFSET, 16,
+ 1, skb->data, skb->len, false);
+ fc_exch_recv(lport, fp);
+}
+
+static void qedf_ll2_process_skb(struct work_struct *work)
+{
+ struct qedf_skb_work *skb_work =
+ container_of(work, struct qedf_skb_work, work);
+ struct qedf_ctx *qedf = skb_work->qedf;
+ struct sk_buff *skb = skb_work->skb;
+ struct ethhdr *eh;
+
+ if (!qedf) {
+ QEDF_ERR(NULL, "qedf is NULL\n");
+ goto err_out;
+ }
+
+ eh = (struct ethhdr *)skb->data;
+
+ /* Undo VLAN encapsulation */
+ if (eh->h_proto == htons(ETH_P_8021Q)) {
+ memmove((u8 *)eh + VLAN_HLEN, eh, ETH_ALEN * 2);
+ eh = (struct ethhdr *)skb_pull(skb, VLAN_HLEN);
+ skb_reset_mac_header(skb);
+ }
+
+ /*
+ * Process either a FIP frame or FCoE frame based on the
+ * protocol value. If it's not either just drop the
+ * frame.
+ */
+ if (eh->h_proto == htons(ETH_P_FIP)) {
+ qedf_fip_recv(qedf, skb);
+ goto out;
+ } else if (eh->h_proto == htons(ETH_P_FCOE)) {
+ __skb_pull(skb, ETH_HLEN);
+ qedf_recv_frame(qedf, skb);
+ goto out;
+ } else
+ goto err_out;
+
+err_out:
+ kfree_skb(skb);
+out:
+ kfree(skb_work);
+ return;
+}
+
+static int qedf_ll2_rx(void *cookie, struct sk_buff *skb,
+ u32 arg1, u32 arg2)
+{
+ struct qedf_ctx *qedf = (struct qedf_ctx *)cookie;
+ struct qedf_skb_work *skb_work;
+
+ skb_work = kzalloc(sizeof(struct qedf_skb_work), GFP_ATOMIC);
+ if (!skb_work) {
+ QEDF_WARN(&(qedf->dbg_ctx), "Could not allocate skb_work so "
+ "dropping frame.\n");
+ kfree_skb(skb);
+ return 0;
+ }
+
+ INIT_WORK(&skb_work->work, qedf_ll2_process_skb);
+ skb_work->skb = skb;
+ skb_work->qedf = qedf;
+ queue_work(qedf->ll2_recv_wq, &skb_work->work);
+
+ return 0;
+}
+
+static struct qed_ll2_cb_ops qedf_ll2_cb_ops = {
+ .rx_cb = qedf_ll2_rx,
+ .tx_cb = NULL,
+};
+
+/* Main thread to process I/O completions */
+void qedf_fp_io_handler(struct work_struct *work)
+{
+ struct qedf_io_work *io_work =
+ container_of(work, struct qedf_io_work, work);
+ u32 comp_type;
+
+ /*
+ * Deferred part of unsolicited CQE sends
+ * frame to libfc.
+ */
+ comp_type = (io_work->cqe.cqe_data >>
+ FCOE_CQE_CQE_TYPE_SHIFT) &
+ FCOE_CQE_CQE_TYPE_MASK;
+ if (comp_type == FCOE_UNSOLIC_CQE_TYPE &&
+ io_work->fp)
+ fc_exch_recv(io_work->qedf->lport, io_work->fp);
+ else
+ qedf_process_cqe(io_work->qedf, &io_work->cqe);
+
+ kfree(io_work);
+}
+
+static int qedf_alloc_and_init_sb(struct qedf_ctx *qedf,
+ struct qed_sb_info *sb_info, u16 sb_id)
+{
+ struct status_block *sb_virt;
+ dma_addr_t sb_phys;
+ int ret;
+
+ sb_virt = dma_alloc_coherent(&qedf->pdev->dev,
+ sizeof(struct status_block), &sb_phys, GFP_KERNEL);
+
+ if (!sb_virt) {
+ QEDF_ERR(&(qedf->dbg_ctx), "Status block allocation failed "
+ "for id = %d.\n", sb_id);
+ return -ENOMEM;
+ }
+
+ ret = qed_ops->common->sb_init(qedf->cdev, sb_info, sb_virt, sb_phys,
+ sb_id, QED_SB_TYPE_STORAGE);
+
+ if (ret) {
+ QEDF_ERR(&(qedf->dbg_ctx), "Status block initialization "
+ "failed for id = %d.\n", sb_id);
+ return ret;
+ }
+
+ return 0;
+}
+
+static void qedf_free_sb(struct qedf_ctx *qedf, struct qed_sb_info *sb_info)
+{
+ if (sb_info->sb_virt)
+ dma_free_coherent(&qedf->pdev->dev, sizeof(*sb_info->sb_virt),
+ (void *)sb_info->sb_virt, sb_info->sb_phys);
+}
+
+static void qedf_destroy_sb(struct qedf_ctx *qedf)
+{
+ int id;
+ struct qedf_fastpath *fp = NULL;
+
+ for (id = 0; id < qedf->num_queues; id++) {
+ fp = &(qedf->fp_array[id]);
+ if (fp->sb_id == QEDF_SB_ID_NULL)
+ break;
+ qedf_free_sb(qedf, fp->sb_info);
+ kfree(fp->sb_info);
+ }
+ kfree(qedf->fp_array);
+}
+
+static int qedf_prepare_sb(struct qedf_ctx *qedf)
+{
+ int id;
+ struct qedf_fastpath *fp;
+ int ret;
+
+ qedf->fp_array =
+ kcalloc(qedf->num_queues, sizeof(struct qedf_fastpath),
+ GFP_KERNEL);
+
+ if (!qedf->fp_array) {
+ QEDF_ERR(&(qedf->dbg_ctx), "fastpath array allocation "
+ "failed.\n");
+ return -ENOMEM;
+ }
+
+ for (id = 0; id < qedf->num_queues; id++) {
+ fp = &(qedf->fp_array[id]);
+ fp->sb_id = QEDF_SB_ID_NULL;
+ fp->sb_info = kcalloc(1, sizeof(*fp->sb_info), GFP_KERNEL);
+ if (!fp->sb_info) {
+ QEDF_ERR(&(qedf->dbg_ctx), "SB info struct "
+ "allocation failed.\n");
+ goto err;
+ }
+ ret = qedf_alloc_and_init_sb(qedf, fp->sb_info, id);
+ if (ret) {
+ QEDF_ERR(&(qedf->dbg_ctx), "SB allocation and "
+ "initialization failed.\n");
+ goto err;
+ }
+ fp->sb_id = id;
+ fp->qedf = qedf;
+ fp->cq_num_entries =
+ qedf->global_queues[id]->cq_mem_size /
+ sizeof(struct fcoe_cqe);
+ }
+err:
+ return 0;
+}
+
+void qedf_process_cqe(struct qedf_ctx *qedf, struct fcoe_cqe *cqe)
+{
+ u16 xid;
+ struct qedf_ioreq *io_req;
+ struct qedf_rport *fcport;
+ u32 comp_type;
+
+ comp_type = (cqe->cqe_data >> FCOE_CQE_CQE_TYPE_SHIFT) &
+ FCOE_CQE_CQE_TYPE_MASK;
+
+ xid = cqe->cqe_data & FCOE_CQE_TASK_ID_MASK;
+ io_req = &qedf->cmd_mgr->cmds[xid];
+
+ /* Completion not for a valid I/O anymore so just return */
+ if (!io_req)
+ return;
+
+ fcport = io_req->fcport;
+
+ if (fcport == NULL) {
+ QEDF_ERR(&(qedf->dbg_ctx), "fcport is NULL.\n");
+ return;
+ }
+
+ /*
+ * Check that fcport is offloaded. If it isn't then the spinlock
+ * isn't valid and shouldn't be taken. We should just return.
+ */
+ if (!test_bit(QEDF_RPORT_SESSION_READY, &fcport->flags)) {
+ QEDF_ERR(&(qedf->dbg_ctx), "Session not offloaded yet.\n");
+ return;
+ }
+
+
+ switch (comp_type) {
+ case FCOE_GOOD_COMPLETION_CQE_TYPE:
+ atomic_inc(&fcport->free_sqes);
+ switch (io_req->cmd_type) {
+ case QEDF_SCSI_CMD:
+ qedf_scsi_completion(qedf, cqe, io_req);
+ break;
+ case QEDF_ELS:
+ qedf_process_els_compl(qedf, cqe, io_req);
+ break;
+ case QEDF_TASK_MGMT_CMD:
+ qedf_process_tmf_compl(qedf, cqe, io_req);
+ break;
+ case QEDF_SEQ_CLEANUP:
+ qedf_process_seq_cleanup_compl(qedf, cqe, io_req);
+ break;
+ }
+ break;
+ case FCOE_ERROR_DETECTION_CQE_TYPE:
+ atomic_inc(&fcport->free_sqes);
+ QEDF_INFO(&(qedf->dbg_ctx), QEDF_LOG_IO,
+ "Error detect CQE.\n");
+ qedf_process_error_detect(qedf, cqe, io_req);
+ break;
+ case FCOE_EXCH_CLEANUP_CQE_TYPE:
+ atomic_inc(&fcport->free_sqes);
+ QEDF_INFO(&(qedf->dbg_ctx), QEDF_LOG_IO,
+ "Cleanup CQE.\n");
+ qedf_process_cleanup_compl(qedf, cqe, io_req);
+ break;
+ case FCOE_ABTS_CQE_TYPE:
+ atomic_inc(&fcport->free_sqes);
+ QEDF_INFO(&(qedf->dbg_ctx), QEDF_LOG_IO,
+ "Abort CQE.\n");
+ qedf_process_abts_compl(qedf, cqe, io_req);
+ break;
+ case FCOE_DUMMY_CQE_TYPE:
+ atomic_inc(&fcport->free_sqes);
+ QEDF_INFO(&(qedf->dbg_ctx), QEDF_LOG_IO,
+ "Dummy CQE.\n");
+ break;
+ case FCOE_LOCAL_COMP_CQE_TYPE:
+ atomic_inc(&fcport->free_sqes);
+ QEDF_INFO(&(qedf->dbg_ctx), QEDF_LOG_IO,
+ "Local completion CQE.\n");
+ break;
+ case FCOE_WARNING_CQE_TYPE:
+ atomic_inc(&fcport->free_sqes);
+ QEDF_INFO(&(qedf->dbg_ctx), QEDF_LOG_IO,
+ "Warning CQE.\n");
+ qedf_process_warning_compl(qedf, cqe, io_req);
+ break;
+ case MAX_FCOE_CQE_TYPE:
+ atomic_inc(&fcport->free_sqes);
+ QEDF_INFO(&(qedf->dbg_ctx), QEDF_LOG_IO,
+ "Max FCoE CQE.\n");
+ break;
+ default:
+ QEDF_INFO(&(qedf->dbg_ctx), QEDF_LOG_IO,
+ "Default CQE.\n");
+ break;
+ }
+}
+
+static void qedf_free_bdq(struct qedf_ctx *qedf)
+{
+ int i;
+
+ if (qedf->bdq_pbl_list)
+ dma_free_coherent(&qedf->pdev->dev, QEDF_PAGE_SIZE,
+ qedf->bdq_pbl_list, qedf->bdq_pbl_list_dma);
+
+ if (qedf->bdq_pbl)
+ dma_free_coherent(&qedf->pdev->dev, qedf->bdq_pbl_mem_size,
+ qedf->bdq_pbl, qedf->bdq_pbl_dma);
+
+ for (i = 0; i < QEDF_BDQ_SIZE; i++) {
+ if (qedf->bdq[i].buf_addr) {
+ dma_free_coherent(&qedf->pdev->dev, QEDF_BDQ_BUF_SIZE,
+ qedf->bdq[i].buf_addr, qedf->bdq[i].buf_dma);
+ }
+ }
+}
+
+static void qedf_free_global_queues(struct qedf_ctx *qedf)
+{
+ int i;
+ struct global_queue **gl = qedf->global_queues;
+
+ for (i = 0; i < qedf->num_queues; i++) {
+ if (!gl[i])
+ continue;
+
+ if (gl[i]->cq)
+ dma_free_coherent(&qedf->pdev->dev,
+ gl[i]->cq_mem_size, gl[i]->cq, gl[i]->cq_dma);
+ if (gl[i]->cq_pbl)
+ dma_free_coherent(&qedf->pdev->dev, gl[i]->cq_pbl_size,
+ gl[i]->cq_pbl, gl[i]->cq_pbl_dma);
+
+ kfree(gl[i]);
+ }
+
+ qedf_free_bdq(qedf);
+}
+
+static int qedf_alloc_bdq(struct qedf_ctx *qedf)
+{
+ int i;
+ struct scsi_bd *pbl;
+ u64 *list;
+ dma_addr_t page;
+
+ /* Alloc dma memory for BDQ buffers */
+ for (i = 0; i < QEDF_BDQ_SIZE; i++) {
+ qedf->bdq[i].buf_addr = dma_alloc_coherent(&qedf->pdev->dev,
+ QEDF_BDQ_BUF_SIZE, &qedf->bdq[i].buf_dma, GFP_KERNEL);
+ if (!qedf->bdq[i].buf_addr) {
+ QEDF_ERR(&(qedf->dbg_ctx), "Could not allocate BDQ "
+ "buffer %d.\n", i);
+ return -ENOMEM;
+ }
+ }
+
+ /* Alloc dma memory for BDQ page buffer list */
+ qedf->bdq_pbl_mem_size =
+ QEDF_BDQ_SIZE * sizeof(struct scsi_bd);
+ qedf->bdq_pbl_mem_size =
+ ALIGN(qedf->bdq_pbl_mem_size, QEDF_PAGE_SIZE);
+
+ qedf->bdq_pbl = dma_alloc_coherent(&qedf->pdev->dev,
+ qedf->bdq_pbl_mem_size, &qedf->bdq_pbl_dma, GFP_KERNEL);
+ if (!qedf->bdq_pbl) {
+ QEDF_ERR(&(qedf->dbg_ctx), "Could not allocate BDQ PBL.\n");
+ return -ENOMEM;
+ }
+
+ QEDF_INFO(&(qedf->dbg_ctx), QEDF_LOG_DISC,
+ "BDQ PBL addr=0x%p dma=0x%llx.\n", qedf->bdq_pbl,
+ qedf->bdq_pbl_dma);
+
+ /*
+ * Populate BDQ PBL with physical and virtual address of individual
+ * BDQ buffers
+ */
+ pbl = (struct scsi_bd *)qedf->bdq_pbl;
+ for (i = 0; i < QEDF_BDQ_SIZE; i++) {
+ pbl->address.hi = cpu_to_le32(U64_HI(qedf->bdq[i].buf_dma));
+ pbl->address.lo = cpu_to_le32(U64_LO(qedf->bdq[i].buf_dma));
+ pbl->opaque.hi = 0;
+ /* Opaque lo data is an index into the BDQ array */
+ pbl->opaque.lo = cpu_to_le32(i);
+ pbl++;
+ }
+
+ /* Allocate list of PBL pages */
+ qedf->bdq_pbl_list = dma_alloc_coherent(&qedf->pdev->dev,
+ QEDF_PAGE_SIZE, &qedf->bdq_pbl_list_dma, GFP_KERNEL);
+ if (!qedf->bdq_pbl_list) {
+ QEDF_ERR(&(qedf->dbg_ctx), "Could not allocate list of PBL "
+ "pages.\n");
+ return -ENOMEM;
+ }
+ memset(qedf->bdq_pbl_list, 0, QEDF_PAGE_SIZE);
+
+ /*
+ * Now populate PBL list with pages that contain pointers to the
+ * individual buffers.
+ */
+ qedf->bdq_pbl_list_num_entries = qedf->bdq_pbl_mem_size /
+ QEDF_PAGE_SIZE;
+ list = (u64 *)qedf->bdq_pbl_list;
+ page = qedf->bdq_pbl_list_dma;
+ for (i = 0; i < qedf->bdq_pbl_list_num_entries; i++) {
+ *list = qedf->bdq_pbl_dma;
+ list++;
+ page += QEDF_PAGE_SIZE;
+ }
+
+ return 0;
+}
+
+static int qedf_alloc_global_queues(struct qedf_ctx *qedf)
+{
+ u32 *list;
+ int i;
+ int status = 0, rc;
+ u32 *pbl;
+ dma_addr_t page;
+ int num_pages;
+
+ /* Allocate and map CQs, RQs */
+ /*
+ * Number of global queues (CQ / RQ). This should
+ * be <= number of available MSIX vectors for the PF
+ */
+ if (!qedf->num_queues) {
+ QEDF_ERR(&(qedf->dbg_ctx), "No MSI-X vectors available!\n");
+ return 1;
+ }
+
+ /*
+ * Make sure we allocated the PBL that will contain the physical
+ * addresses of our queues
+ */
+ if (!qedf->p_cpuq) {
+ status = 1;
+ goto mem_alloc_failure;
+ }
+
+ qedf->global_queues = kzalloc((sizeof(struct global_queue *)
+ * qedf->num_queues), GFP_KERNEL);
+ if (!qedf->global_queues) {
+ QEDF_ERR(&(qedf->dbg_ctx), "Unable to allocate global "
+ "queues array ptr memory\n");
+ return -ENOMEM;
+ }
+ QEDF_INFO(&(qedf->dbg_ctx), QEDF_LOG_DISC,
+ "qedf->global_queues=%p.\n", qedf->global_queues);
+
+ /* Allocate DMA coherent buffers for BDQ */
+ rc = qedf_alloc_bdq(qedf);
+ if (rc)
+ goto mem_alloc_failure;
+
+ /* Allocate a CQ and an associated PBL for each MSI-X vector */
+ for (i = 0; i < qedf->num_queues; i++) {
+ qedf->global_queues[i] = kzalloc(sizeof(struct global_queue),
+ GFP_KERNEL);
+ if (!qedf->global_queues[i]) {
+ QEDF_WARN(&(qedf->dbg_ctx), "Unable to allocation "
+ "global queue %d.\n", i);
+ goto mem_alloc_failure;
+ }
+
+ qedf->global_queues[i]->cq_mem_size =
+ FCOE_PARAMS_CQ_NUM_ENTRIES * sizeof(struct fcoe_cqe);
+ qedf->global_queues[i]->cq_mem_size =
+ ALIGN(qedf->global_queues[i]->cq_mem_size, QEDF_PAGE_SIZE);
+
+ qedf->global_queues[i]->cq_pbl_size =
+ (qedf->global_queues[i]->cq_mem_size /
+ PAGE_SIZE) * sizeof(void *);
+ qedf->global_queues[i]->cq_pbl_size =
+ ALIGN(qedf->global_queues[i]->cq_pbl_size, QEDF_PAGE_SIZE);
+
+ qedf->global_queues[i]->cq =
+ dma_alloc_coherent(&qedf->pdev->dev,
+ qedf->global_queues[i]->cq_mem_size,
+ &qedf->global_queues[i]->cq_dma, GFP_KERNEL);
+
+ if (!qedf->global_queues[i]->cq) {
+ QEDF_WARN(&(qedf->dbg_ctx), "Could not allocate "
+ "cq.\n");
+ status = -ENOMEM;
+ goto mem_alloc_failure;
+ }
+ memset(qedf->global_queues[i]->cq, 0,
+ qedf->global_queues[i]->cq_mem_size);
+
+ qedf->global_queues[i]->cq_pbl =
+ dma_alloc_coherent(&qedf->pdev->dev,
+ qedf->global_queues[i]->cq_pbl_size,
+ &qedf->global_queues[i]->cq_pbl_dma, GFP_KERNEL);
+
+ if (!qedf->global_queues[i]->cq_pbl) {
+ QEDF_WARN(&(qedf->dbg_ctx), "Could not allocate "
+ "cq PBL.\n");
+ status = -ENOMEM;
+ goto mem_alloc_failure;
+ }
+ memset(qedf->global_queues[i]->cq_pbl, 0,
+ qedf->global_queues[i]->cq_pbl_size);
+
+ /* Create PBL */
+ num_pages = qedf->global_queues[i]->cq_mem_size /
+ QEDF_PAGE_SIZE;
+ page = qedf->global_queues[i]->cq_dma;
+ pbl = (u32 *)qedf->global_queues[i]->cq_pbl;
+
+ while (num_pages--) {
+ *pbl = U64_LO(page);
+ pbl++;
+ *pbl = U64_HI(page);
+ pbl++;
+ page += QEDF_PAGE_SIZE;
+ }
+ /* Set the initial consumer index for cq */
+ qedf->global_queues[i]->cq_cons_idx = 0;
+ }
+
+ list = (u32 *)qedf->p_cpuq;
+
+ /*
+ * The list is built as follows: CQ#0 PBL pointer, RQ#0 PBL pointer,
+ * CQ#1 PBL pointer, RQ#1 PBL pointer, etc. Each PBL pointer points
+ * to the physical address which contains an array of pointers to
+ * the physical addresses of the specific queue pages.
+ */
+ for (i = 0; i < qedf->num_queues; i++) {
+ *list = U64_LO(qedf->global_queues[i]->cq_pbl_dma);
+ list++;
+ *list = U64_HI(qedf->global_queues[i]->cq_pbl_dma);
+ list++;
+ *list = U64_LO(0);
+ list++;
+ *list = U64_HI(0);
+ list++;
+ }
+
+ return 0;
+
+mem_alloc_failure:
+ qedf_free_global_queues(qedf);
+ return status;
+}
+
+static int qedf_set_fcoe_pf_param(struct qedf_ctx *qedf)
+{
+ u8 sq_num_pbl_pages;
+ u32 sq_mem_size;
+ u32 cq_mem_size;
+ u32 cq_num_entries;
+ int rval;
+
+ /*
+ * The number of completion queues/fastpath interrupts/status blocks
+ * we allocation is the minimum off:
+ *
+ * Number of CPUs
+ * Number of MSI-X vectors
+ * Max number allocated in hardware (QEDF_MAX_NUM_CQS)
+ */
+ qedf->num_queues = min((unsigned int)QEDF_MAX_NUM_CQS,
+ num_online_cpus());
+
+ QEDF_INFO(&(qedf->dbg_ctx), QEDF_LOG_DISC, "Number of CQs is %d.\n",
+ qedf->num_queues);
+
+ qedf->p_cpuq = pci_alloc_consistent(qedf->pdev,
+ qedf->num_queues * sizeof(struct qedf_glbl_q_params),
+ &qedf->hw_p_cpuq);
+
+ if (!qedf->p_cpuq) {
+ QEDF_ERR(&(qedf->dbg_ctx), "pci_alloc_consistent failed.\n");
+ return 1;
+ }
+
+ rval = qedf_alloc_global_queues(qedf);
+ if (rval) {
+ QEDF_ERR(&(qedf->dbg_ctx), "Global queue allocation "
+ "failed.\n");
+ return 1;
+ }
+
+ /* Calculate SQ PBL size in the same manner as in qedf_sq_alloc() */
+ sq_mem_size = SQ_NUM_ENTRIES * sizeof(struct fcoe_wqe);
+ sq_mem_size = ALIGN(sq_mem_size, QEDF_PAGE_SIZE);
+ sq_num_pbl_pages = (sq_mem_size / QEDF_PAGE_SIZE);
+
+ /* Calculate CQ num entries */
+ cq_mem_size = FCOE_PARAMS_CQ_NUM_ENTRIES * sizeof(struct fcoe_cqe);
+ cq_mem_size = ALIGN(cq_mem_size, QEDF_PAGE_SIZE);
+ cq_num_entries = cq_mem_size / sizeof(struct fcoe_cqe);
+
+ memset(&(qedf->pf_params), 0,
+ sizeof(qedf->pf_params));
+
+ /* Setup the value for fcoe PF */
+ qedf->pf_params.fcoe_pf_params.num_cons = QEDF_MAX_SESSIONS;
+ qedf->pf_params.fcoe_pf_params.num_tasks = FCOE_PARAMS_NUM_TASKS;
+ qedf->pf_params.fcoe_pf_params.glbl_q_params_addr =
+ (u64)qedf->hw_p_cpuq;
+ qedf->pf_params.fcoe_pf_params.sq_num_pbl_pages = sq_num_pbl_pages;
+
+ qedf->pf_params.fcoe_pf_params.rq_buffer_log_size = 0;
+
+ qedf->pf_params.fcoe_pf_params.cq_num_entries = cq_num_entries;
+ qedf->pf_params.fcoe_pf_params.num_cqs = qedf->num_queues;
+
+ /* log_page_size: 12 for 4KB pages */
+ qedf->pf_params.fcoe_pf_params.log_page_size = ilog2(QEDF_PAGE_SIZE);
+
+ qedf->pf_params.fcoe_pf_params.mtu = 9000;
+ qedf->pf_params.fcoe_pf_params.gl_rq_pi = QEDF_FCOE_PARAMS_GL_RQ_PI;
+ qedf->pf_params.fcoe_pf_params.gl_cmd_pi = QEDF_FCOE_PARAMS_GL_CMD_PI;
+
+ /* BDQ address and size */
+ qedf->pf_params.fcoe_pf_params.bdq_pbl_base_addr[0] =
+ qedf->bdq_pbl_list_dma;
+ qedf->pf_params.fcoe_pf_params.bdq_pbl_num_entries[0] =
+ qedf->bdq_pbl_list_num_entries;
+ qedf->pf_params.fcoe_pf_params.rq_buffer_size = QEDF_BDQ_BUF_SIZE;
+
+ QEDF_INFO(&(qedf->dbg_ctx), QEDF_LOG_DISC,
+ "bdq_list=%p bdq_pbl_list_dma=%llx bdq_pbl_list_entries=%d.\n",
+ qedf->bdq_pbl_list,
+ qedf->pf_params.fcoe_pf_params.bdq_pbl_base_addr[0],
+ qedf->pf_params.fcoe_pf_params.bdq_pbl_num_entries[0]);
+
+ QEDF_INFO(&(qedf->dbg_ctx), QEDF_LOG_DISC,
+ "cq_num_entries=%d.\n",
+ qedf->pf_params.fcoe_pf_params.cq_num_entries);
+
+ return 0;
+}
+
+/* Free DMA coherent memory for array of queue pointers we pass to qed */
+static void qedf_free_fcoe_pf_param(struct qedf_ctx *qedf)
+{
+ size_t size = 0;
+
+ if (qedf->p_cpuq) {
+ size = qedf->num_queues * sizeof(struct qedf_glbl_q_params);
+ pci_free_consistent(qedf->pdev, size, qedf->p_cpuq,
+ qedf->hw_p_cpuq);
+ }
+
+ qedf_free_global_queues(qedf);
+
+ if (qedf->global_queues)
+ kfree(qedf->global_queues);
+}
+
+/*
+ * PCI driver functions
+ */
+
+static const struct pci_device_id qedf_pci_tbl[] = {
+ { PCI_DEVICE(PCI_VENDOR_ID_QLOGIC, 0x165c) },
+ { PCI_DEVICE(PCI_VENDOR_ID_QLOGIC, 0x8080) },
+ {0}
+};
+MODULE_DEVICE_TABLE(pci, qedf_pci_tbl);
+
+static struct pci_driver qedf_pci_driver = {
+ .name = QEDF_MODULE_NAME,
+ .id_table = qedf_pci_tbl,
+ .probe = qedf_probe,
+ .remove = qedf_remove,
+};
+
+static int __qedf_probe(struct pci_dev *pdev, int mode)
+{
+ int rc = -EINVAL;
+ struct fc_lport *lport;
+ struct qedf_ctx *qedf;
+ struct Scsi_Host *host;
+ bool is_vf = false;
+ struct qed_ll2_params params;
+ char host_buf[20];
+ struct qed_link_params link_params;
+ int status;
+ void *task_start, *task_end;
+ struct qed_slowpath_params slowpath_params;
+ struct qed_probe_params qed_params;
+ u16 tmp;
+
+ /*
+ * When doing error recovery we didn't reap the lport so don't try
+ * to reallocate it.
+ */
+ if (mode != QEDF_MODE_RECOVERY) {
+ lport = libfc_host_alloc(&qedf_host_template,
+ sizeof(struct qedf_ctx));
+
+ if (!lport) {
+ QEDF_ERR(NULL, "Could not allocate lport.\n");
+ rc = -ENOMEM;
+ goto err0;
+ }
+
+ /* Initialize qedf_ctx */
+ qedf = lport_priv(lport);
+ qedf->lport = lport;
+ qedf->ctlr.lp = lport;
+ qedf->pdev = pdev;
+ qedf->dbg_ctx.pdev = pdev;
+ qedf->dbg_ctx.host_no = lport->host->host_no;
+ spin_lock_init(&qedf->hba_lock);
+ INIT_LIST_HEAD(&qedf->fcports);
+ qedf->curr_conn_id = QEDF_MAX_SESSIONS - 1;
+ atomic_set(&qedf->num_offloads, 0);
+ qedf->stop_io_on_error = false;
+ pci_set_drvdata(pdev, qedf);
+
+ QEDF_INFO(&(qedf->dbg_ctx), QEDF_LOG_INFO,
+ "QLogic FastLinQ FCoE Module qedf %s, "
+ "FW %d.%d.%d.%d\n", QEDF_VERSION,
+ FW_MAJOR_VERSION, FW_MINOR_VERSION, FW_REVISION_VERSION,
+ FW_ENGINEERING_VERSION);
+ } else {
+ /* Init pointers during recovery */
+ qedf = pci_get_drvdata(pdev);
+ lport = qedf->lport;
+ }
+
+ host = lport->host;
+
+ /* Allocate mempool for qedf_io_work structs */
+ qedf->io_mempool = mempool_create_slab_pool(QEDF_IO_WORK_MIN,
+ qedf_io_work_cache);
+ if (qedf->io_mempool == NULL) {
+ QEDF_ERR(&(qedf->dbg_ctx), "qedf->io_mempool is NULL.\n");
+ goto err1;
+ }
+ QEDF_INFO(&(qedf->dbg_ctx), QEDF_LOG_INFO, "qedf->io_mempool=%p.\n",
+ qedf->io_mempool);
+
+ sprintf(host_buf, "qedf_%u_link",
+ qedf->lport->host->host_no);
+ qedf->link_update_wq = create_singlethread_workqueue(host_buf);
+ INIT_DELAYED_WORK(&qedf->link_update, qedf_handle_link_update);
+ INIT_DELAYED_WORK(&qedf->link_recovery, qedf_link_recovery);
+
+ qedf->fipvlan_retries = qedf_fipvlan_retries;
+
+ /*
+ * Common probe. Takes care of basic hardware init and pci_*
+ * functions.
+ */
+ memset(&qed_params, 0, sizeof(qed_params));
+ qed_params.protocol = QED_PROTOCOL_FCOE;
+ qed_params.dp_module = qedf_dp_module;
+ qed_params.dp_level = qedf_dp_level;
+ qed_params.is_vf = is_vf;
+ qedf->cdev = qed_ops->common->probe(pdev, &qed_params);
+ if (!qedf->cdev) {
+ rc = -ENODEV;
+ goto err1;
+ }
+
+ /* queue allocation code should come here
+ * order should be
+ * slowpath_start
+ * status block allocation
+ * interrupt registration (to get min number of queues)
+ * set_fcoe_pf_param
+ * qed_sp_fcoe_func_start
+ */
+ rc = qedf_set_fcoe_pf_param(qedf);
+ if (rc) {
+ QEDF_ERR(&(qedf->dbg_ctx), "Cannot set fcoe pf param.\n");
+ goto err2;
+ }
+ qed_ops->common->update_pf_params(qedf->cdev, &qedf->pf_params);
+
+ /* Learn information crucial for qedf to progress */
+ rc = qed_ops->fill_dev_info(qedf->cdev, &qedf->dev_info);
+ if (rc) {
+ QEDF_ERR(&(qedf->dbg_ctx), "Failed to dev info.\n");
+ goto err1;
+ }
+
+ /* Record BDQ producer doorbell addresses */
+ qedf->bdq_primary_prod = qedf->dev_info.primary_dbq_rq_addr;
+ qedf->bdq_secondary_prod = qedf->dev_info.secondary_bdq_rq_addr;
+ QEDF_INFO(&(qedf->dbg_ctx), QEDF_LOG_DISC,
+ "BDQ primary_prod=%p secondary_prod=%p.\n", qedf->bdq_primary_prod,
+ qedf->bdq_secondary_prod);
+
+ qed_ops->register_ops(qedf->cdev, &qedf_cb_ops, qedf);
+
+ rc = qedf_prepare_sb(qedf);
+ if (rc) {
+
+ QEDF_ERR(&(qedf->dbg_ctx), "Cannot start slowpath.\n");
+ goto err2;
+ }
+
+ /* Start the Slowpath-process */
+ slowpath_params.int_mode = QED_INT_MODE_MSIX;
+ slowpath_params.drv_major = QEDF_DRIVER_MAJOR_VER;
+ slowpath_params.drv_minor = QEDF_DRIVER_MINOR_VER;
+ slowpath_params.drv_rev = QEDF_DRIVER_REV_VER;
+ slowpath_params.drv_eng = QEDF_DRIVER_ENG_VER;
+ memcpy(slowpath_params.name, "qedf", QED_DRV_VER_STR_SIZE);
+ rc = qed_ops->common->slowpath_start(qedf->cdev, &slowpath_params);
+ if (rc) {
+ QEDF_ERR(&(qedf->dbg_ctx), "Cannot start slowpath.\n");
+ goto err2;
+ }
+
+ /*
+ * update_pf_params needs to be called before and after slowpath
+ * start
+ */
+ qed_ops->common->update_pf_params(qedf->cdev, &qedf->pf_params);
+
+ /* Setup interrupts */
+ rc = qedf_setup_int(qedf);
+ if (rc)
+ goto err3;
+
+ rc = qed_ops->start(qedf->cdev, &qedf->tasks);
+ if (rc) {
+ QEDF_ERR(&(qedf->dbg_ctx), "Cannot start FCoE function.\n");
+ goto err4;
+ }
+ task_start = qedf_get_task_mem(&qedf->tasks, 0);
+ task_end = qedf_get_task_mem(&qedf->tasks, MAX_TID_BLOCKS_FCOE - 1);
+ QEDF_INFO(&(qedf->dbg_ctx), QEDF_LOG_DISC, "Task context start=%p, "
+ "end=%p block_size=%u.\n", task_start, task_end,
+ qedf->tasks.size);
+
+ /*
+ * We need to write the number of BDs in the BDQ we've preallocated so
+ * the f/w will do a prefetch and we'll get an unsolicited CQE when a
+ * packet arrives.
+ */
+ qedf->bdq_prod_idx = QEDF_BDQ_SIZE;
+ QEDF_INFO(&(qedf->dbg_ctx), QEDF_LOG_DISC,
+ "Writing %d to primary and secondary BDQ doorbell registers.\n",
+ qedf->bdq_prod_idx);
+ writew(qedf->bdq_prod_idx, qedf->bdq_primary_prod);
+ tmp = readw(qedf->bdq_primary_prod);
+ writew(qedf->bdq_prod_idx, qedf->bdq_secondary_prod);
+ tmp = readw(qedf->bdq_secondary_prod);
+
+ qed_ops->common->set_power_state(qedf->cdev, PCI_D0);
+
+ /* Now that the dev_info struct has been filled in set the MAC
+ * address
+ */
+ ether_addr_copy(qedf->mac, qedf->dev_info.common.hw_mac);
+ QEDF_INFO(&(qedf->dbg_ctx), QEDF_LOG_DISC, "MAC address is %pM.\n",
+ qedf->mac);
+
+ /* Set the WWNN and WWPN based on the MAC address */
+ qedf->wwnn = fcoe_wwn_from_mac(qedf->mac, 1, 0);
+ qedf->wwpn = fcoe_wwn_from_mac(qedf->mac, 2, 0);
+ QEDF_INFO(&(qedf->dbg_ctx), QEDF_LOG_DISC, "WWNN=%016llx "
+ "WWPN=%016llx.\n", qedf->wwnn, qedf->wwpn);
+
+ sprintf(host_buf, "host_%d", host->host_no);
+ qed_ops->common->set_id(qedf->cdev, host_buf, QEDF_VERSION);
+
+
+ /* Set xid max values */
+ qedf->max_scsi_xid = QEDF_MAX_SCSI_XID;
+ qedf->max_els_xid = QEDF_MAX_ELS_XID;
+
+ /* Allocate cmd mgr */
+ qedf->cmd_mgr = qedf_cmd_mgr_alloc(qedf);
+ if (!qedf->cmd_mgr) {
+ QEDF_ERR(&(qedf->dbg_ctx), "Failed to allocate cmd mgr.\n");
+ goto err5;
+ }
+
+ if (mode != QEDF_MODE_RECOVERY) {
+ host->transportt = qedf_fc_transport_template;
+ host->can_queue = QEDF_MAX_ELS_XID;
+ host->max_lun = qedf_max_lun;
+ host->max_cmd_len = QEDF_MAX_CDB_LEN;
+ rc = scsi_add_host(host, &pdev->dev);
+ if (rc)
+ goto err6;
+ }
+
+ memset(¶ms, 0, sizeof(params));
+ params.mtu = 9000;
+ ether_addr_copy(params.ll2_mac_address, qedf->mac);
+
+ /* Start LL2 processing thread */
+ snprintf(host_buf, 20, "qedf_%d_ll2", host->host_no);
+ qedf->ll2_recv_wq =
+ create_singlethread_workqueue(host_buf);
+ if (!qedf->ll2_recv_wq) {
+ QEDF_ERR(&(qedf->dbg_ctx), "Failed to LL2 workqueue.\n");
+ goto err7;
+ }
+
+#ifdef CONFIG_DEBUG_FS
+ qedf_dbg_host_init(&(qedf->dbg_ctx), &qedf_debugfs_ops,
+ &qedf_dbg_fops);
+#endif
+
+ /* Start LL2 */
+ qed_ops->ll2->register_cb_ops(qedf->cdev, &qedf_ll2_cb_ops, qedf);
+ rc = qed_ops->ll2->start(qedf->cdev, ¶ms);
+ if (rc) {
+ QEDF_ERR(&(qedf->dbg_ctx), "Could not start Light L2.\n");
+ goto err7;
+ }
+ set_bit(QEDF_LL2_STARTED, &qedf->flags);
+
+ /* hw will be insterting vlan tag*/
+ qedf->vlan_hw_insert = 1;
+ qedf->vlan_id = 0;
+
+ /*
+ * No need to setup fcoe_ctlr or fc_lport objects during recovery since
+ * they were not reaped during the unload process.
+ */
+ if (mode != QEDF_MODE_RECOVERY) {
+ /* Setup imbedded fcoe controller */
+ qedf_fcoe_ctlr_setup(qedf);
+
+ /* Setup lport */
+ rc = qedf_lport_setup(qedf);
+ if (rc) {
+ QEDF_ERR(&(qedf->dbg_ctx),
+ "qedf_lport_setup failed.\n");
+ goto err7;
+ }
+ }
+
+ sprintf(host_buf, "qedf_%u_timer", qedf->lport->host->host_no);
+ qedf->timer_work_queue =
+ create_singlethread_workqueue(host_buf);
+ if (!qedf->timer_work_queue) {
+ QEDF_ERR(&(qedf->dbg_ctx), "Failed to start timer "
+ "workqueue.\n");
+ goto err7;
+ }
+
+ /* DPC workqueue is not reaped during recovery unload */
+ if (mode != QEDF_MODE_RECOVERY) {
+ sprintf(host_buf, "qedf_%u_dpc",
+ qedf->lport->host->host_no);
+ qedf->dpc_wq = create_singlethread_workqueue(host_buf);
+ }
+
+ /*
+ * GRC dump and sysfs parameters are not reaped during the recovery
+ * unload process.
+ */
+ if (mode != QEDF_MODE_RECOVERY) {
+ qedf->grcdump_size = qed_ops->common->dbg_grc_size(qedf->cdev);
+ if (qedf->grcdump_size) {
+ rc = qedf_alloc_grc_dump_buf(&qedf->grcdump,
+ qedf->grcdump_size);
+ if (rc) {
+ QEDF_ERR(&(qedf->dbg_ctx),
+ "GRC Dump buffer alloc failed.\n");
+ qedf->grcdump = NULL;
+ }
+
+ QEDF_INFO(&(qedf->dbg_ctx), QEDF_LOG_DISC,
+ "grcdump: addr=%p, size=%u.\n",
+ qedf->grcdump, qedf->grcdump_size);
+ }
+ qedf_create_sysfs_ctx_attr(qedf);
+
+ /* Initialize I/O tracing for this adapter */
+ spin_lock_init(&qedf->io_trace_lock);
+ qedf->io_trace_idx = 0;
+ }
+
+ init_completion(&qedf->flogi_compl);
+
+ memset(&link_params, 0, sizeof(struct qed_link_params));
+ link_params.link_up = true;
+ status = qed_ops->common->set_link(qedf->cdev, &link_params);
+ if (status)
+ QEDF_WARN(&(qedf->dbg_ctx), "set_link failed.\n");
+
+ /* Start/restart discovery */
+ if (mode == QEDF_MODE_RECOVERY)
+ fcoe_ctlr_link_up(&qedf->ctlr);
+ else
+ fc_fabric_login(lport);
+
+ /* All good */
+ return 0;
+
+err7:
+ if (qedf->ll2_recv_wq)
+ destroy_workqueue(qedf->ll2_recv_wq);
+ fc_remove_host(qedf->lport->host);
+ scsi_remove_host(qedf->lport->host);
+#ifdef CONFIG_DEBUG_FS
+ qedf_dbg_host_exit(&(qedf->dbg_ctx));
+#endif
+err6:
+ qedf_cmd_mgr_free(qedf->cmd_mgr);
+err5:
+ qed_ops->stop(qedf->cdev);
+err4:
+ qedf_free_fcoe_pf_param(qedf);
+ qedf_sync_free_irqs(qedf);
+err3:
+ qed_ops->common->slowpath_stop(qedf->cdev);
+err2:
+ qed_ops->common->remove(qedf->cdev);
+err1:
+ scsi_host_put(lport->host);
+err0:
+ return rc;
+}
+
+static int qedf_probe(struct pci_dev *pdev, const struct pci_device_id *id)
+{
+ return __qedf_probe(pdev, QEDF_MODE_NORMAL);
+}
+
+static void __qedf_remove(struct pci_dev *pdev, int mode)
+{
+ struct qedf_ctx *qedf;
+
+ if (!pdev) {
+ QEDF_ERR(NULL, "pdev is NULL.\n");
+ return;
+ }
+
+ qedf = pci_get_drvdata(pdev);
+
+ /*
+ * Prevent race where we're in board disable work and then try to
+ * rmmod the module.
+ */
+ if (test_bit(QEDF_UNLOADING, &qedf->flags)) {
+ QEDF_ERR(&qedf->dbg_ctx, "Already removing PCI function.\n");
+ return;
+ }
+
+ if (mode != QEDF_MODE_RECOVERY)
+ set_bit(QEDF_UNLOADING, &qedf->flags);
+
+ /* Logoff the fabric to upload all connections */
+ if (mode == QEDF_MODE_RECOVERY)
+ fcoe_ctlr_link_down(&qedf->ctlr);
+ else
+ fc_fabric_logoff(qedf->lport);
+ qedf_wait_for_upload(qedf);
+
+#ifdef CONFIG_DEBUG_FS
+ qedf_dbg_host_exit(&(qedf->dbg_ctx));
+#endif
+
+ /* Stop any link update handling */
+ cancel_delayed_work_sync(&qedf->link_update);
+ destroy_workqueue(qedf->link_update_wq);
+ qedf->link_update_wq = NULL;
+
+ if (qedf->timer_work_queue)
+ destroy_workqueue(qedf->timer_work_queue);
+
+ /* Stop Light L2 */
+ clear_bit(QEDF_LL2_STARTED, &qedf->flags);
+ qed_ops->ll2->stop(qedf->cdev);
+ if (qedf->ll2_recv_wq)
+ destroy_workqueue(qedf->ll2_recv_wq);
+
+ /* Stop fastpath */
+ qedf_sync_free_irqs(qedf);
+ qedf_destroy_sb(qedf);
+
+ /*
+ * During recovery don't destroy OS constructs that represent the
+ * physical port.
+ */
+ if (mode != QEDF_MODE_RECOVERY) {
+ qedf_free_grc_dump_buf(&qedf->grcdump);
+ qedf_remove_sysfs_ctx_attr(qedf);
+
+ /* Remove all SCSI/libfc/libfcoe structures */
+ fcoe_ctlr_destroy(&qedf->ctlr);
+ fc_lport_destroy(qedf->lport);
+ fc_remove_host(qedf->lport->host);
+ scsi_remove_host(qedf->lport->host);
+ }
+
+ qedf_cmd_mgr_free(qedf->cmd_mgr);
+
+ if (mode != QEDF_MODE_RECOVERY) {
+ fc_exch_mgr_free(qedf->lport);
+ fc_lport_free_stats(qedf->lport);
+
+ /* Wait for all vports to be reaped */
+ qedf_wait_for_vport_destroy(qedf);
+ }
+
+ /*
+ * Now that all connections have been uploaded we can stop the
+ * rest of the qed operations
+ */
+ qed_ops->stop(qedf->cdev);
+
+ if (mode != QEDF_MODE_RECOVERY) {
+ if (qedf->dpc_wq) {
+ /* Stop general DPC handling */
+ destroy_workqueue(qedf->dpc_wq);
+ qedf->dpc_wq = NULL;
+ }
+ }
+
+ /* Final shutdown for the board */
+ qedf_free_fcoe_pf_param(qedf);
+ if (mode != QEDF_MODE_RECOVERY) {
+ qed_ops->common->set_power_state(qedf->cdev, PCI_D0);
+ pci_set_drvdata(pdev, NULL);
+ }
+ qed_ops->common->slowpath_stop(qedf->cdev);
+ qed_ops->common->remove(qedf->cdev);
+
+ mempool_destroy(qedf->io_mempool);
+
+ /* Only reap the Scsi_host on a real removal */
+ if (mode != QEDF_MODE_RECOVERY)
+ scsi_host_put(qedf->lport->host);
+}
+
+static void qedf_remove(struct pci_dev *pdev)
+{
+ /* Check to make sure this function wasn't already disabled */
+ if (!atomic_read(&pdev->enable_cnt))
+ return;
+
+ __qedf_remove(pdev, QEDF_MODE_NORMAL);
+}
+
+/*
+ * Module Init/Remove
+ */
+
+static int __init qedf_init(void)
+{
+ int ret;
+
+ /* If debug=1 passed, set the default log mask */
+ if (qedf_debug == QEDF_LOG_DEFAULT)
+ qedf_debug = QEDF_DEFAULT_LOG_MASK;
+
+ /* Print driver banner */
+ QEDF_INFO(NULL, QEDF_LOG_INFO, "%s v%s.\n", QEDF_DESCR,
+ QEDF_VERSION);
+
+ /* Create kmem_cache for qedf_io_work structs */
+ qedf_io_work_cache = kmem_cache_create("qedf_io_work_cache",
+ sizeof(struct qedf_io_work), 0, SLAB_HWCACHE_ALIGN, NULL);
+ if (qedf_io_work_cache == NULL) {
+ QEDF_ERR(NULL, "qedf_io_work_cache is NULL.\n");
+ goto err1;
+ }
+ QEDF_INFO(NULL, QEDF_LOG_DISC, "qedf_io_work_cache=%p.\n",
+ qedf_io_work_cache);
+
+ qed_ops = qed_get_fcoe_ops();
+ if (!qed_ops) {
+ QEDF_ERR(NULL, "Failed to get qed fcoe operations\n");
+ goto err1;
+ }
+
+#ifdef CONFIG_DEBUG_FS
+ qedf_dbg_init("qedf");
+#endif
+
+ qedf_fc_transport_template =
+ fc_attach_transport(&qedf_fc_transport_fn);
+ if (!qedf_fc_transport_template) {
+ QEDF_ERR(NULL, "Could not register with FC transport\n");
+ goto err2;
+ }
+
+ qedf_fc_vport_transport_template =
+ fc_attach_transport(&qedf_fc_vport_transport_fn);
+ if (!qedf_fc_vport_transport_template) {
+ QEDF_ERR(NULL, "Could not register vport template with FC "
+ "transport\n");
+ goto err3;
+ }
+
+ qedf_io_wq = create_workqueue("qedf_io_wq");
+ if (!qedf_io_wq) {
+ QEDF_ERR(NULL, "Could not create qedf_io_wq.\n");
+ goto err4;
+ }
+
+ qedf_cb_ops.get_login_failures = qedf_get_login_failures;
+
+ ret = pci_register_driver(&qedf_pci_driver);
+ if (ret) {
+ QEDF_ERR(NULL, "Failed to register driver\n");
+ goto err5;
+ }
+
+ return 0;
+
+err5:
+ destroy_workqueue(qedf_io_wq);
+err4:
+ fc_release_transport(qedf_fc_vport_transport_template);
+err3:
+ fc_release_transport(qedf_fc_transport_template);
+err2:
+#ifdef CONFIG_DEBUG_FS
+ qedf_dbg_exit();
+#endif
+ qed_put_fcoe_ops();
+err1:
+ return -EINVAL;
+}
+
+static void __exit qedf_cleanup(void)
+{
+ pci_unregister_driver(&qedf_pci_driver);
+
+ destroy_workqueue(qedf_io_wq);
+
+ fc_release_transport(qedf_fc_vport_transport_template);
+ fc_release_transport(qedf_fc_transport_template);
+#ifdef CONFIG_DEBUG_FS
+ qedf_dbg_exit();
+#endif
+ qed_put_fcoe_ops();
+
+ kmem_cache_destroy(qedf_io_work_cache);
+}
+
+MODULE_LICENSE("GPL");
+MODULE_DESCRIPTION("QLogic QEDF 25/40/50/100Gb FCoE Driver");
+MODULE_AUTHOR("QLogic Corporation");
+MODULE_VERSION(QEDF_VERSION);
+module_init(qedf_init);
+module_exit(qedf_cleanup);
--- /dev/null
+/*
+ * QLogic FCoE Offload Driver
+ * Copyright (c) 2016 Cavium Inc.
+ *
+ * This software is available under the terms of the GNU General Public License
+ * (GPL) Version 2, available from the file COPYING in the main directory of
+ * this source tree.
+ */
+
+#define QEDF_VERSION "8.10.7.0"
+#define QEDF_DRIVER_MAJOR_VER 8
+#define QEDF_DRIVER_MINOR_VER 10
+#define QEDF_DRIVER_REV_VER 7
+#define QEDF_DRIVER_ENG_VER 0
+
iscsi_block_session(session->cls_session);
rval = qedi_cleanup_all_io(qedi, qedi_conn, qedi_cmd->task, true);
if (rval) {
- clear_bit(QEDI_CONN_FW_CLEANUP, &qedi_conn->flags);
qedi_clear_task_idx(qedi, qedi_cmd->task_id);
iscsi_unblock_session(session->cls_session);
- return;
+ goto exit_tmf_resp;
}
iscsi_unblock_session(session->cls_session);
spin_lock(&session->back_lock);
__iscsi_complete_pdu(conn, (struct iscsi_hdr *)resp_hdr_ptr, NULL, 0);
spin_unlock(&session->back_lock);
+
+exit_tmf_resp:
kfree(resp_hdr_ptr);
clear_bit(QEDI_CONN_FW_CLEANUP, &qedi_conn->flags);
}
clear_bit(vha->vp_idx, ha->vp_idx_map);
mutex_unlock(&ha->vport_lock);
+ dma_free_coherent(&ha->pdev->dev, vha->gnl.size, vha->gnl.l,
+ vha->gnl.ldma);
+
if (vha->qpair->vp_idx == vha->vp_idx) {
if (qla2xxx_delete_qpair(vha, vha->qpair) != QLA_SUCCESS)
ql_log(ql_log_warn, vha, 0x7087,
/* BSG support for ELS/CT pass through */
void
-qla2x00_bsg_job_done(void *data, void *ptr, int res)
+qla2x00_bsg_job_done(void *ptr, int res)
{
- srb_t *sp = (srb_t *)ptr;
- struct scsi_qla_host *vha = (scsi_qla_host_t *)data;
+ srb_t *sp = ptr;
struct bsg_job *bsg_job = sp->u.bsg_job;
struct fc_bsg_reply *bsg_reply = bsg_job->reply;
bsg_reply->result = res;
bsg_job_done(bsg_job, bsg_reply->result,
bsg_reply->reply_payload_rcv_len);
- sp->free(vha, sp);
+ sp->free(sp);
}
void
-qla2x00_bsg_sp_free(void *data, void *ptr)
+qla2x00_bsg_sp_free(void *ptr)
{
- srb_t *sp = (srb_t *)ptr;
- struct scsi_qla_host *vha = sp->fcport->vha;
+ srb_t *sp = ptr;
+ struct qla_hw_data *ha = sp->vha->hw;
struct bsg_job *bsg_job = sp->u.bsg_job;
struct fc_bsg_request *bsg_request = bsg_job->request;
-
- struct qla_hw_data *ha = vha->hw;
struct qla_mt_iocb_rqst_fx00 *piocb_rqst;
if (sp->type == SRB_FXIOCB_BCMD) {
sp->type == SRB_FXIOCB_BCMD ||
sp->type == SRB_ELS_CMD_HST)
kfree(sp->fcport);
- qla2x00_rel_sp(vha, sp);
+ qla2x00_rel_sp(sp);
}
int
if (rval != QLA_SUCCESS) {
ql_log(ql_log_warn, vha, 0x700e,
"qla2x00_start_sp failed = %d\n", rval);
- qla2x00_rel_sp(vha, sp);
+ qla2x00_rel_sp(sp);
rval = -EIO;
goto done_unmap_sg;
}
if (rval != QLA_SUCCESS) {
ql_log(ql_log_warn, vha, 0x7017,
"qla2x00_start_sp failed=%d.\n", rval);
- qla2x00_rel_sp(vha, sp);
+ qla2x00_rel_sp(sp);
rval = -EIO;
goto done_free_fcport;
}
done:
spin_unlock_irqrestore(&ha->hardware_lock, flags);
- sp->free(vha, sp);
+ sp->free(sp);
return 0;
}
#include "qla_settings.h"
+#define MODE_DUAL (MODE_TARGET | MODE_INITIATOR)
+
/*
* Data bit definitions
*/
#define MAX_CMDSZ 16 /* SCSI maximum CDB size. */
#include "qla_fw.h"
+
+struct name_list_extended {
+ struct get_name_list_extended *l;
+ dma_addr_t ldma;
+ struct list_head fcports; /* protect by sess_list */
+ u32 size;
+ u8 sent;
+};
/*
* Timeout timer counts in seconds
*/
uint8_t wwpn[WWN_SIZE];
};
+struct ct_arg {
+ void *iocb;
+ u16 nport_handle;
+ dma_addr_t req_dma;
+ dma_addr_t rsp_dma;
+ u32 req_size;
+ u32 rsp_size;
+ void *req;
+ void *rsp;
+};
+
/*
* SRB extensions.
*/
#define SRB_LOGIN_COND_PLOGI BIT_1
#define SRB_LOGIN_SKIP_PRLI BIT_2
uint16_t data[2];
+ u32 iop[2];
} logio;
struct {
#define ELS_DCMD_TIMEOUT 20
__le16 comp_status;
struct completion comp;
} abt;
+ struct ct_arg ctarg;
+ struct {
+ __le16 in_mb[28]; /* fr fw */
+ __le16 out_mb[28]; /* to fw */
+ void *out, *in;
+ dma_addr_t out_dma, in_dma;
+ } mbx;
+ struct {
+ struct imm_ntfy_from_isp *ntfy;
+ } nack;
} u;
struct timer_list timer;
#define SRB_FXIOCB_BCMD 11
#define SRB_ABT_CMD 12
#define SRB_ELS_DCMD 13
+#define SRB_MB_IOCB 14
+#define SRB_CT_PTHRU_CMD 15
+#define SRB_NACK_PLOGI 16
+#define SRB_NACK_PRLI 17
+#define SRB_NACK_LOGO 18
typedef struct srb {
atomic_t ref_count;
struct fc_port *fcport;
+ struct scsi_qla_host *vha;
uint32_t handle;
uint16_t flags;
uint16_t type;
char *name;
int iocbs;
struct qla_qpair *qpair;
+ u32 gen1; /* scratch */
+ u32 gen2; /* scratch */
union {
struct srb_iocb iocb_cmd;
struct bsg_job *bsg_job;
struct srb_cmd scmd;
} u;
- void (*done)(void *, void *, int);
- void (*free)(void *, void *);
+ void (*done)(void *, int);
+ void (*free)(void *);
} srb_t;
#define GET_CMD_SP(sp) (sp->u.scmd.cmd)
#define SS_RESIDUAL_OVER BIT_10
#define SS_SENSE_LEN_VALID BIT_9
#define SS_RESPONSE_INFO_LEN_VALID BIT_8
+#define SS_SCSI_STATUS_BYTE 0xff
#define SS_RESERVE_CONFLICT (BIT_4 | BIT_3)
#define SS_BUSY_CONDITION BIT_3
uint8_t port_name[WWN_SIZE];
};
+#ifndef IMMED_NOTIFY_TYPE
+#define IMMED_NOTIFY_TYPE 0x0D /* Immediate notify entry. */
+/*
+ * ISP queue - immediate notify entry structure definition.
+ * This is sent by the ISP to the Target driver.
+ * This IOCB would have report of events sent by the
+ * initiator, that needs to be handled by the target
+ * driver immediately.
+ */
+struct imm_ntfy_from_isp {
+ uint8_t entry_type; /* Entry type. */
+ uint8_t entry_count; /* Entry count. */
+ uint8_t sys_define; /* System defined. */
+ uint8_t entry_status; /* Entry Status. */
+ union {
+ struct {
+ uint32_t sys_define_2; /* System defined. */
+ target_id_t target;
+ uint16_t lun;
+ uint8_t target_id;
+ uint8_t reserved_1;
+ uint16_t status_modifier;
+ uint16_t status;
+ uint16_t task_flags;
+ uint16_t seq_id;
+ uint16_t srr_rx_id;
+ uint32_t srr_rel_offs;
+ uint16_t srr_ui;
+#define SRR_IU_DATA_IN 0x1
+#define SRR_IU_DATA_OUT 0x5
+#define SRR_IU_STATUS 0x7
+ uint16_t srr_ox_id;
+ uint8_t reserved_2[28];
+ } isp2x;
+ struct {
+ uint32_t reserved;
+ uint16_t nport_handle;
+ uint16_t reserved_2;
+ uint16_t flags;
+#define NOTIFY24XX_FLAGS_GLOBAL_TPRLO BIT_1
+#define NOTIFY24XX_FLAGS_PUREX_IOCB BIT_0
+ uint16_t srr_rx_id;
+ uint16_t status;
+ uint8_t status_subcode;
+ uint8_t fw_handle;
+ uint32_t exchange_address;
+ uint32_t srr_rel_offs;
+ uint16_t srr_ui;
+ uint16_t srr_ox_id;
+ union {
+ struct {
+ uint8_t node_name[8];
+ } plogi; /* PLOGI/ADISC/PDISC */
+ struct {
+ /* PRLI word 3 bit 0-15 */
+ uint16_t wd3_lo;
+ uint8_t resv0[6];
+ } prli;
+ struct {
+ uint8_t port_id[3];
+ uint8_t resv1;
+ uint16_t nport_handle;
+ uint16_t resv2;
+ } req_els;
+ } u;
+ uint8_t port_name[8];
+ uint8_t resv3[3];
+ uint8_t vp_index;
+ uint32_t reserved_5;
+ uint8_t port_id[3];
+ uint8_t reserved_6;
+ } isp24;
+ } u;
+ uint16_t reserved_7;
+ uint16_t ox_id;
+} __packed;
+#endif
+
/*
* ISP request and response queue entry sizes
*/
#define FC4_TYPE_OTHER 0x0
#define FC4_TYPE_UNKNOWN 0xff
+/* mailbox command 4G & above */
+struct mbx_24xx_entry {
+ uint8_t entry_type;
+ uint8_t entry_count;
+ uint8_t sys_define1;
+ uint8_t entry_status;
+ uint32_t handle;
+ uint16_t mb[28];
+};
+
+#define IOCB_SIZE 64
+
/*
* Fibre channel port type.
*/
- typedef enum {
+typedef enum {
FCT_UNKNOWN,
FCT_RSCN,
FCT_SWITCH,
FCT_TARGET
} fc_port_type_t;
+enum qla_sess_deletion {
+ QLA_SESS_DELETION_NONE = 0,
+ QLA_SESS_DELETION_IN_PROGRESS,
+ QLA_SESS_DELETED,
+};
+
+enum qlt_plogi_link_t {
+ QLT_PLOGI_LINK_SAME_WWN,
+ QLT_PLOGI_LINK_CONFLICT,
+ QLT_PLOGI_LINK_MAX
+};
+
+struct qlt_plogi_ack_t {
+ struct list_head list;
+ struct imm_ntfy_from_isp iocb;
+ port_id_t id;
+ int ref_count;
+ void *fcport;
+};
+
+struct ct_sns_desc {
+ struct ct_sns_pkt *ct_sns;
+ dma_addr_t ct_sns_dma;
+};
+
+enum discovery_state {
+ DSC_DELETED,
+ DSC_GID_PN,
+ DSC_GNL,
+ DSC_LOGIN_PEND,
+ DSC_LOGIN_FAILED,
+ DSC_GPDB,
+ DSC_GPSC,
+ DSC_UPD_FCPORT,
+ DSC_LOGIN_COMPLETE,
+ DSC_DELETE_PEND,
+};
+
+enum login_state { /* FW control Target side */
+ DSC_LS_LLIOCB_SENT = 2,
+ DSC_LS_PLOGI_PEND,
+ DSC_LS_PLOGI_COMP,
+ DSC_LS_PRLI_PEND,
+ DSC_LS_PRLI_COMP,
+ DSC_LS_PORT_UNAVAIL,
+ DSC_LS_PRLO_PEND = 9,
+ DSC_LS_LOGO_PEND,
+};
+
+enum fcport_mgt_event {
+ FCME_RELOGIN = 1,
+ FCME_RSCN,
+ FCME_GIDPN_DONE,
+ FCME_PLOGI_DONE, /* Initiator side sent LLIOCB */
+ FCME_GNL_DONE,
+ FCME_GPSC_DONE,
+ FCME_GPDB_DONE,
+ FCME_GPNID_DONE,
+ FCME_DELETE_DONE,
+};
+
+enum rscn_addr_format {
+ RSCN_PORT_ADDR,
+ RSCN_AREA_ADDR,
+ RSCN_DOM_ADDR,
+ RSCN_FAB_ADDR,
+};
+
/*
* Fibre channel port structure.
*/
uint16_t loop_id;
uint16_t old_loop_id;
+ unsigned int conf_compl_supported:1;
+ unsigned int deleted:2;
+ unsigned int local:1;
+ unsigned int logout_on_delete:1;
+ unsigned int logo_ack_needed:1;
+ unsigned int keep_nport_handle:1;
+ unsigned int send_els_logo:1;
+ unsigned int login_pause:1;
+ unsigned int login_succ:1;
+
+ struct fc_port *conflict;
+ unsigned char logout_completed;
+ int generation;
+
+ struct se_session *se_sess;
+ struct kref sess_kref;
+ struct qla_tgt *tgt;
+ unsigned long expires;
+ struct list_head del_list_entry;
+ struct work_struct free_work;
+
+ struct qlt_plogi_ack_t *plogi_link[QLT_PLOGI_LINK_MAX];
+
uint16_t tgt_id;
uint16_t old_tgt_id;
unsigned long retry_delay_timestamp;
struct qla_tgt_sess *tgt_session;
+ struct ct_sns_desc ct_desc;
+ enum discovery_state disc_state;
+ enum login_state fw_login_state;
+ u32 login_gen, last_login_gen;
+ u32 rscn_gen, last_rscn_gen;
+ u32 chip_reset;
+ struct list_head gnl_entry;
+ struct work_struct del_work;
+ u8 iocb[IOCB_SIZE];
} fc_port_t;
+#define QLA_FCPORT_SCAN 1
+#define QLA_FCPORT_FOUND 2
+
+struct event_arg {
+ enum fcport_mgt_event event;
+ fc_port_t *fcport;
+ srb_t *sp;
+ port_id_t id;
+ u16 data[2], rc;
+ u8 port_name[WWN_SIZE];
+ u32 iop[2];
+};
+
#include "qla_mr.h"
/*
#define GFT_ID_REQ_SIZE (16 + 4)
#define GFT_ID_RSP_SIZE (16 + 32)
+#define GID_PN_CMD 0x121
+#define GID_PN_REQ_SIZE (16 + 8)
+#define GID_PN_RSP_SIZE (16 + 4)
+
#define RFT_ID_CMD 0x217
#define RFT_ID_REQ_SIZE (16 + 4 + 32)
#define RFT_ID_RSP_SIZE 16
uint8_t reserved;
uint8_t port_name[3];
} gff_id;
+
+ struct {
+ uint8_t port_name[8];
+ } gid_pn;
} req;
};
struct {
uint8_t fc4_features[128];
} gff_id;
+ struct {
+ uint8_t reserved;
+ uint8_t port_id[3];
+ } gid_pn;
} rsp;
};
uint16_t (*calc_req_entries) (uint16_t);
void (*build_iocbs) (srb_t *, cmd_entry_t *, uint16_t);
- void * (*prep_ms_iocb) (struct scsi_qla_host *, uint32_t, uint32_t);
- void * (*prep_ms_fdmi_iocb) (struct scsi_qla_host *, uint32_t,
+ void *(*prep_ms_iocb) (struct scsi_qla_host *, struct ct_arg *);
+ void *(*prep_ms_fdmi_iocb) (struct scsi_qla_host *, uint32_t,
uint32_t);
- uint8_t * (*read_nvram) (struct scsi_qla_host *, uint8_t *,
+ uint8_t *(*read_nvram) (struct scsi_qla_host *, uint8_t *,
uint32_t, uint32_t);
int (*write_nvram) (struct scsi_qla_host *, uint8_t *, uint32_t,
uint32_t);
QLA_EVT_AEN,
QLA_EVT_IDC_ACK,
QLA_EVT_ASYNC_LOGIN,
- QLA_EVT_ASYNC_LOGIN_DONE,
QLA_EVT_ASYNC_LOGOUT,
QLA_EVT_ASYNC_LOGOUT_DONE,
QLA_EVT_ASYNC_ADISC,
QLA_EVT_ASYNC_ADISC_DONE,
QLA_EVT_UEVENT,
QLA_EVT_AENFX,
+ QLA_EVT_GIDPN,
+ QLA_EVT_GPNID,
+ QLA_EVT_GPNID_DONE,
+ QLA_EVT_NEW_SESS,
+ QLA_EVT_GPDB,
+ QLA_EVT_GPSC,
+ QLA_EVT_UPD_FCPORT,
+ QLA_EVT_GNL,
+ QLA_EVT_NACK,
};
struct {
srb_t *sp;
} iosb;
+ struct {
+ port_id_t id;
+ } gpnid;
+ struct {
+ port_id_t id;
+ u8 port_name[8];
+ void *pla;
+ } new_sess;
+ struct { /*Get PDB, Get Speed, update fcport, gnl, gidpn */
+ fc_port_t *fcport;
+ u8 opt;
+ } fcport;
+ struct {
+ fc_port_t *fcport;
+ u8 iocb[IOCB_SIZE];
+ int type;
+ } nack;
} u;
};
struct qla_hw_data *hw;
struct work_struct q_work;
struct list_head qp_list_elem; /* vha->qp_list */
+ struct scsi_qla_host *vha;
};
/* Place holder for FW buffer parameters */
/* Protected by hw lock */
uint32_t enable_class_2:1;
uint32_t enable_explicit_conf:1;
- uint32_t ini_mode_force_reverse:1;
uint32_t node_name_set:1;
dma_addr_t atio_dma; /* Physical address. */
#define FLOGI_SP_SUPPORT BIT_13
uint8_t port_no; /* Physical port of adapter */
+ uint8_t exch_starvation;
/* Timeout timers. */
uint8_t loop_down_abort_time; /* port down timer */
#define FCOE_CTX_RESET_NEEDED 18 /* Initiate FCoE context reset */
#define MPI_RESET_NEEDED 19 /* Initiate MPI FW reset */
#define ISP_QUIESCE_NEEDED 20 /* Driver need some quiescence */
-#define SCR_PENDING 21 /* SCR in target mode */
+#define FREE_BIT 21
#define PORT_UPDATE_NEEDED 22
#define FX00_RESET_RECOVERY 23
#define FX00_TARGET_SCAN 24
/* list of commands waiting on workqueue */
struct list_head qla_cmd_list;
struct list_head qla_sess_op_cmd_list;
+ struct list_head unknown_atio_list;
spinlock_t cmd_list_lock;
+ struct delayed_work unknown_atio_work;
/* Counter to detect races between ELS and RSCN events */
atomic_t generation_tick;
struct qla8044_reset_template reset_tmplt;
struct qla_tgt_counters tgt_counters;
uint16_t bbcr;
+ struct name_list_extended gnl;
+ /* Count of active session/fcport */
+ int fcport_count;
+ wait_queue_head_t fcport_waitQ;
} scsi_qla_host_t;
struct qla27xx_image_status {
scsi_qla_host_t *vha = s->private;
struct qla_hw_data *ha = vha->hw;
unsigned long flags;
- struct qla_tgt_sess *sess = NULL;
+ struct fc_port *sess = NULL;
struct qla_tgt *tgt= vha->vha_tgt.qla_tgt;
seq_printf(s, "%s\n",vha->host_str);
seq_printf(s, "Port ID Port Name Handle\n");
spin_lock_irqsave(&ha->tgt.sess_lock, flags);
- list_for_each_entry(sess, &tgt->sess_list, sess_list_entry) {
+ list_for_each_entry(sess, &vha->vp_fcports, list)
seq_printf(s, "%02x:%02x:%02x %8phC %d\n",
- sess->s_id.b.domain,sess->s_id.b.area,
- sess->s_id.b.al_pa, sess->port_name,
- sess->loop_id);
- }
+ sess->d_id.b.domain, sess->d_id.b.area,
+ sess->d_id.b.al_pa, sess->port_name,
+ sess->loop_id);
spin_unlock_irqrestore(&ha->tgt.sess_lock, flags);
}
uint8_t reserved_3[24];
};
+/*
+ * MB 75h returns a list of DB entries similar to port_database_24xx(64B).
+ * However, in this case it returns 1st 40 bytes.
+ */
+struct get_name_list_extended {
+ __le16 flags;
+ u8 current_login_state;
+ u8 last_login_state;
+ u8 hard_address[3];
+ u8 reserved_1;
+ u8 port_id[3];
+ u8 sequence_id;
+ __le16 port_timer;
+ __le16 nport_handle; /* N_PORT handle. */
+ __le16 receive_data_size;
+ __le16 reserved_2;
+
+ /* PRLI SVC Param are Big endian */
+ u8 prli_svc_param_word_0[2]; /* Bits 15-0 of word 0 */
+ u8 prli_svc_param_word_3[2]; /* Bits 15-0 of word 3 */
+ u8 port_name[WWN_SIZE];
+ u8 node_name[WWN_SIZE];
+};
+
+/* MB 75h: This is the short version of the database */
+struct get_name_list {
+ u8 port_node_name[WWN_SIZE]; /* B7 most sig, B0 least sig */
+ __le16 nport_handle;
+ u8 reserved;
+};
+
struct vp_database_24xx {
uint16_t vp_status;
uint8_t options;
};
#define VP_RPT_ID_IOCB_TYPE 0x32 /* Report ID Acquisition entry. */
+enum VP_STATUS {
+ VP_STAT_COMPL,
+ VP_STAT_FAIL,
+ VP_STAT_ID_CHG,
+ VP_STAT_SNS_TO, /* timeout */
+ VP_STAT_SNS_RJT,
+ VP_STAT_SCR_TO, /* timeout */
+ VP_STAT_SCR_RJT,
+};
+
+enum VP_FLAGS {
+ VP_FLAGS_CON_FLOOP = 1,
+ VP_FLAGS_CON_P2P = 2,
+ VP_FLAGS_CON_FABRIC = 3,
+ VP_FLAGS_NAME_VALID = BIT_5,
+};
+
struct vp_rpt_id_entry_24xx {
uint8_t entry_type; /* Entry type. */
uint8_t entry_count; /* Entry count. */
uint8_t sys_define; /* System defined. */
uint8_t entry_status; /* Entry Status. */
-
- uint32_t handle; /* System handle. */
-
- uint16_t vp_count; /* Format 0 -- | VP setup | VP acq |. */
- /* Format 1 -- | VP count |. */
- uint16_t vp_idx; /* Format 0 -- Reserved. */
- /* Format 1 -- VP status and index. */
+ uint32_t resv1;
+ uint8_t vp_acquired;
+ uint8_t vp_setup;
+ uint8_t vp_idx; /* Format 0=reserved */
+ uint8_t vp_status; /* Format 0=reserved */
uint8_t port_id[3];
uint8_t format;
-
- uint8_t vp_idx_map[16];
-
- uint8_t reserved_4[24];
- uint16_t bbcr;
- uint8_t reserved_5[6];
+ union {
+ struct {
+ /* format 0 loop */
+ uint8_t vp_idx_map[16];
+ uint8_t reserved_4[32];
+ } f0;
+ struct {
+ /* format 1 fabric */
+ uint8_t vpstat1_subcode; /* vp_status=1 subcode */
+ uint8_t flags;
+ uint16_t fip_flags;
+ uint8_t rsv2[12];
+
+ uint8_t ls_rjt_vendor;
+ uint8_t ls_rjt_explanation;
+ uint8_t ls_rjt_reason;
+ uint8_t rsv3[5];
+
+ uint8_t port_name[8];
+ uint8_t node_name[8];
+ uint16_t bbcr;
+ uint8_t reserved_5[6];
+ } f1;
+ struct { /* format 2: N2N direct connect */
+ uint8_t vpstat1_subcode;
+ uint8_t flags;
+ uint16_t rsv6;
+ uint8_t rsv2[12];
+
+ uint8_t ls_rjt_vendor;
+ uint8_t ls_rjt_explanation;
+ uint8_t ls_rjt_reason;
+ uint8_t rsv3[5];
+
+ uint8_t port_name[8];
+ uint8_t node_name[8];
+ uint32_t remote_nport_id;
+ uint32_t reserved_5;
+ } f2;
+ } u;
};
#define VF_EVFP_IOCB_TYPE 0x26 /* Exchange Virtual Fabric Parameters entry. */
uint16_t *);
extern void qla2x00_async_adisc_done(struct scsi_qla_host *, fc_port_t *,
uint16_t *);
+struct qla_work_evt *qla2x00_alloc_work(struct scsi_qla_host *,
+ enum qla_work_type);
+extern int qla24xx_async_gnl(struct scsi_qla_host *, fc_port_t *);
+int qla2x00_post_work(struct scsi_qla_host *vha, struct qla_work_evt *e);
extern void *qla2x00_alloc_iocbs(struct scsi_qla_host *, srb_t *);
extern void *qla2x00_alloc_iocbs_ready(struct scsi_qla_host *, srb_t *);
extern int qla24xx_update_fcport_fcp_prio(scsi_qla_host_t *, fc_port_t *);
extern struct qla_qpair *qla2xxx_create_qpair(struct scsi_qla_host *,
int, int);
extern int qla2xxx_delete_qpair(struct scsi_qla_host *, struct qla_qpair *);
+void qla2x00_fcport_event_handler(scsi_qla_host_t *, struct event_arg *);
+int qla24xx_async_gpdb(struct scsi_qla_host *, fc_port_t *, u8);
+int qla24xx_async_notify_ack(scsi_qla_host_t *, fc_port_t *,
+ struct imm_ntfy_from_isp *, int);
+int qla24xx_post_newsess_work(struct scsi_qla_host *, port_id_t *, u8 *,
+ void *);
+int qla24xx_fcport_handle_login(struct scsi_qla_host *, fc_port_t *);
/*
* Global Data in qla_os.c source file.
extern int ql2xexlogins;
extern int ql2xexchoffld;
extern int ql2xfwholdabts;
+extern int ql2xmvasynctoatio;
extern int qla2x00_loop_reset(scsi_qla_host_t *);
extern void qla2x00_abort_all_cmds(scsi_qla_host_t *, int);
extern int qla2x00_post_idc_ack_work(struct scsi_qla_host *, uint16_t *);
extern int qla2x00_post_async_login_work(struct scsi_qla_host *, fc_port_t *,
uint16_t *);
-extern int qla2x00_post_async_login_done_work(struct scsi_qla_host *,
- fc_port_t *, uint16_t *);
extern int qla2x00_post_async_logout_work(struct scsi_qla_host *, fc_port_t *,
uint16_t *);
extern int qla2x00_post_async_logout_done_work(struct scsi_qla_host *,
extern int qla2x00_post_uevent_work(struct scsi_qla_host *, u32);
extern void qla2x00_disable_board_on_pci_error(struct work_struct *);
-extern void qla2x00_sp_compl(void *, void *, int);
-extern void qla2xxx_qpair_sp_free_dma(void *, void *);
-extern void qla2xxx_qpair_sp_compl(void *, void *, int);
+extern void qla2x00_sp_compl(void *, int);
+extern void qla2xxx_qpair_sp_free_dma(void *);
+extern void qla2xxx_qpair_sp_compl(void *, int);
+extern int qla24xx_post_upd_fcport_work(struct scsi_qla_host *, fc_port_t *);
+void qla2x00_handle_login_done_event(struct scsi_qla_host *, fc_port_t *,
+ uint16_t *);
+int qla24xx_post_gnl_work(struct scsi_qla_host *, fc_port_t *);
/*
* Global Functions in qla_mid.c source file.
extern int qla24xx_vport_create_req_sanity_check(struct fc_vport *);
extern scsi_qla_host_t * qla24xx_create_vhost(struct fc_vport *);
-extern void qla2x00_sp_free_dma(void *, void *);
+extern void qla2x00_sp_free_dma(void *);
extern char *qla2x00_get_fw_version_str(struct scsi_qla_host *, char *);
extern void qla2x00_mark_device_lost(scsi_qla_host_t *, fc_port_t *, int, int);
extern int
qla2x00_init_firmware(scsi_qla_host_t *, uint16_t);
-extern int
-qla2x00_get_node_name_list(scsi_qla_host_t *, void **, int *);
-
extern int
qla2x00_get_port_database(scsi_qla_host_t *, fc_port_t *, uint8_t);
uint32_t);
extern irqreturn_t
qla2xxx_msix_rsp_q(int irq, void *dev_id);
+fc_port_t *qla2x00_find_fcport_by_loopid(scsi_qla_host_t *, uint16_t);
+fc_port_t *qla2x00_find_fcport_by_wwpn(scsi_qla_host_t *, u8 *, u8);
+fc_port_t *qla2x00_find_fcport_by_nportid(scsi_qla_host_t *, port_id_t *, u8);
/*
* Global Function Prototypes in qla_sup.c source file.
/*
* Global Function Prototypes in qla_gs.c source file.
*/
-extern void *qla2x00_prep_ms_iocb(scsi_qla_host_t *, uint32_t, uint32_t);
-extern void *qla24xx_prep_ms_iocb(scsi_qla_host_t *, uint32_t, uint32_t);
+extern void *qla2x00_prep_ms_iocb(scsi_qla_host_t *, struct ct_arg *);
+extern void *qla24xx_prep_ms_iocb(scsi_qla_host_t *, struct ct_arg *);
extern int qla2x00_ga_nxt(scsi_qla_host_t *, fc_port_t *);
extern int qla2x00_gid_pt(scsi_qla_host_t *, sw_info_t *);
extern int qla2x00_gpn_id(scsi_qla_host_t *, sw_info_t *);
extern int qla2x00_gfpn_id(scsi_qla_host_t *, sw_info_t *);
extern int qla2x00_gpsc(scsi_qla_host_t *, sw_info_t *);
extern void qla2x00_get_sym_node_name(scsi_qla_host_t *, uint8_t *, size_t);
+extern int qla2x00_chk_ms_status(scsi_qla_host_t *, ms_iocb_entry_t *,
+ struct ct_sns_rsp *, const char *);
+extern void qla2x00_async_iocb_timeout(void *data);
+extern int qla24xx_async_gidpn(scsi_qla_host_t *, fc_port_t *);
+int qla24xx_post_gidpn_work(struct scsi_qla_host *, fc_port_t *);
+void qla24xx_handle_gidpn_event(scsi_qla_host_t *, struct event_arg *);
+
+extern void qla2x00_free_fcport(fc_port_t *);
+
+extern int qla24xx_post_gpnid_work(struct scsi_qla_host *, port_id_t *);
+extern int qla24xx_async_gpnid(scsi_qla_host_t *, port_id_t *);
+void qla24xx_async_gpnid_done(scsi_qla_host_t *, srb_t*);
+void qla24xx_handle_gpnid_event(scsi_qla_host_t *, struct event_arg *);
+
+int qla24xx_post_gpsc_work(struct scsi_qla_host *, fc_port_t *);
+int qla24xx_async_gpsc(scsi_qla_host_t *, fc_port_t *);
+int qla2x00_mgmt_svr_login(scsi_qla_host_t *);
/*
* Global Function Prototypes in qla_attr.c source file.
/* IOCB related functions */
extern int qla82xx_start_scsi(srb_t *);
-extern void qla2x00_sp_free(void *, void *);
+extern void qla2x00_sp_free(void *);
extern void qla2x00_sp_timeout(unsigned long);
-extern void qla2x00_bsg_job_done(void *, void *, int);
-extern void qla2x00_bsg_sp_free(void *, void *);
+extern void qla2x00_bsg_job_done(void *, int);
+extern void qla2x00_bsg_sp_free(void *);
extern void qla2x00_start_iocbs(struct scsi_qla_host *, struct req_que *);
/* Interrupt related */
extern int qla_set_exchoffld_mem_cfg(scsi_qla_host_t *, dma_addr_t);
extern void qlt_handle_abts_recv(struct scsi_qla_host *, response_t *);
+int qla24xx_async_notify_ack(scsi_qla_host_t *, fc_port_t *,
+ struct imm_ntfy_from_isp *, int);
+void qla24xx_do_nack_work(struct scsi_qla_host *, struct qla_work_evt *);
+void qlt_plogi_ack_link(struct scsi_qla_host *, struct qlt_plogi_ack_t *,
+ struct fc_port *, enum qlt_plogi_link_t);
+void qlt_plogi_ack_unref(struct scsi_qla_host *, struct qlt_plogi_ack_t *);
+extern void qlt_schedule_sess_for_deletion(struct fc_port *, bool);
+extern void qlt_schedule_sess_for_deletion_lock(struct fc_port *);
+extern struct fc_port *qlt_find_sess_invalidate_other(scsi_qla_host_t *,
+ uint64_t wwn, port_id_t port_id, uint16_t loop_id, struct fc_port **);
+void qla24xx_delete_sess_fn(struct work_struct *);
+void qlt_unknown_atio_work_fn(struct work_struct *);
+
#endif /* _QLA_GBL_H */
* Returns a pointer to the @ha's ms_iocb.
*/
void *
-qla2x00_prep_ms_iocb(scsi_qla_host_t *vha, uint32_t req_size, uint32_t rsp_size)
+qla2x00_prep_ms_iocb(scsi_qla_host_t *vha, struct ct_arg *arg)
{
struct qla_hw_data *ha = vha->hw;
ms_iocb_entry_t *ms_pkt;
- ms_pkt = ha->ms_iocb;
+ ms_pkt = (ms_iocb_entry_t *)arg->iocb;
memset(ms_pkt, 0, sizeof(ms_iocb_entry_t));
ms_pkt->entry_type = MS_IOCB_TYPE;
ms_pkt->timeout = cpu_to_le16(ha->r_a_tov / 10 * 2);
ms_pkt->cmd_dsd_count = cpu_to_le16(1);
ms_pkt->total_dsd_count = cpu_to_le16(2);
- ms_pkt->rsp_bytecount = cpu_to_le32(rsp_size);
- ms_pkt->req_bytecount = cpu_to_le32(req_size);
+ ms_pkt->rsp_bytecount = cpu_to_le32(arg->rsp_size);
+ ms_pkt->req_bytecount = cpu_to_le32(arg->req_size);
- ms_pkt->dseg_req_address[0] = cpu_to_le32(LSD(ha->ct_sns_dma));
- ms_pkt->dseg_req_address[1] = cpu_to_le32(MSD(ha->ct_sns_dma));
+ ms_pkt->dseg_req_address[0] = cpu_to_le32(LSD(arg->req_dma));
+ ms_pkt->dseg_req_address[1] = cpu_to_le32(MSD(arg->req_dma));
ms_pkt->dseg_req_length = ms_pkt->req_bytecount;
- ms_pkt->dseg_rsp_address[0] = cpu_to_le32(LSD(ha->ct_sns_dma));
- ms_pkt->dseg_rsp_address[1] = cpu_to_le32(MSD(ha->ct_sns_dma));
+ ms_pkt->dseg_rsp_address[0] = cpu_to_le32(LSD(arg->rsp_dma));
+ ms_pkt->dseg_rsp_address[1] = cpu_to_le32(MSD(arg->rsp_dma));
ms_pkt->dseg_rsp_length = ms_pkt->rsp_bytecount;
vha->qla_stats.control_requests++;
* Returns a pointer to the @ha's ms_iocb.
*/
void *
-qla24xx_prep_ms_iocb(scsi_qla_host_t *vha, uint32_t req_size, uint32_t rsp_size)
+qla24xx_prep_ms_iocb(scsi_qla_host_t *vha, struct ct_arg *arg)
{
struct qla_hw_data *ha = vha->hw;
struct ct_entry_24xx *ct_pkt;
- ct_pkt = (struct ct_entry_24xx *)ha->ms_iocb;
+ ct_pkt = (struct ct_entry_24xx *)arg->iocb;
memset(ct_pkt, 0, sizeof(struct ct_entry_24xx));
ct_pkt->entry_type = CT_IOCB_TYPE;
ct_pkt->entry_count = 1;
- ct_pkt->nport_handle = cpu_to_le16(NPH_SNS);
+ ct_pkt->nport_handle = cpu_to_le16(arg->nport_handle);
ct_pkt->timeout = cpu_to_le16(ha->r_a_tov / 10 * 2);
ct_pkt->cmd_dsd_count = cpu_to_le16(1);
ct_pkt->rsp_dsd_count = cpu_to_le16(1);
- ct_pkt->rsp_byte_count = cpu_to_le32(rsp_size);
- ct_pkt->cmd_byte_count = cpu_to_le32(req_size);
+ ct_pkt->rsp_byte_count = cpu_to_le32(arg->rsp_size);
+ ct_pkt->cmd_byte_count = cpu_to_le32(arg->req_size);
- ct_pkt->dseg_0_address[0] = cpu_to_le32(LSD(ha->ct_sns_dma));
- ct_pkt->dseg_0_address[1] = cpu_to_le32(MSD(ha->ct_sns_dma));
+ ct_pkt->dseg_0_address[0] = cpu_to_le32(LSD(arg->req_dma));
+ ct_pkt->dseg_0_address[1] = cpu_to_le32(MSD(arg->req_dma));
ct_pkt->dseg_0_len = ct_pkt->cmd_byte_count;
- ct_pkt->dseg_1_address[0] = cpu_to_le32(LSD(ha->ct_sns_dma));
- ct_pkt->dseg_1_address[1] = cpu_to_le32(MSD(ha->ct_sns_dma));
+ ct_pkt->dseg_1_address[0] = cpu_to_le32(LSD(arg->rsp_dma));
+ ct_pkt->dseg_1_address[1] = cpu_to_le32(MSD(arg->rsp_dma));
ct_pkt->dseg_1_len = ct_pkt->rsp_byte_count;
ct_pkt->vp_index = vha->vp_idx;
return &p->p.req;
}
-static int
+int
qla2x00_chk_ms_status(scsi_qla_host_t *vha, ms_iocb_entry_t *ms_pkt,
struct ct_sns_rsp *ct_rsp, const char *routine)
{
struct ct_sns_req *ct_req;
struct ct_sns_rsp *ct_rsp;
struct qla_hw_data *ha = vha->hw;
+ struct ct_arg arg;
if (IS_QLA2100(ha) || IS_QLA2200(ha))
return qla2x00_sns_ga_nxt(vha, fcport);
+ arg.iocb = ha->ms_iocb;
+ arg.req_dma = ha->ct_sns_dma;
+ arg.rsp_dma = ha->ct_sns_dma;
+ arg.req_size = GA_NXT_REQ_SIZE;
+ arg.rsp_size = GA_NXT_RSP_SIZE;
+ arg.nport_handle = NPH_SNS;
+
/* Issue GA_NXT */
/* Prepare common MS IOCB */
- ms_pkt = ha->isp_ops->prep_ms_iocb(vha, GA_NXT_REQ_SIZE,
- GA_NXT_RSP_SIZE);
+ ms_pkt = ha->isp_ops->prep_ms_iocb(vha, &arg);
/* Prepare CT request */
ct_req = qla2x00_prep_ct_req(ha->ct_sns, GA_NXT_CMD,
struct ct_sns_gid_pt_data *gid_data;
struct qla_hw_data *ha = vha->hw;
uint16_t gid_pt_rsp_size;
+ struct ct_arg arg;
if (IS_QLA2100(ha) || IS_QLA2200(ha))
return qla2x00_sns_gid_pt(vha, list);
gid_data = NULL;
gid_pt_rsp_size = qla2x00_gid_pt_rsp_size(vha);
+
+ arg.iocb = ha->ms_iocb;
+ arg.req_dma = ha->ct_sns_dma;
+ arg.rsp_dma = ha->ct_sns_dma;
+ arg.req_size = GID_PT_REQ_SIZE;
+ arg.rsp_size = gid_pt_rsp_size;
+ arg.nport_handle = NPH_SNS;
+
/* Issue GID_PT */
/* Prepare common MS IOCB */
- ms_pkt = ha->isp_ops->prep_ms_iocb(vha, GID_PT_REQ_SIZE,
- gid_pt_rsp_size);
+ ms_pkt = ha->isp_ops->prep_ms_iocb(vha, &arg);
/* Prepare CT request */
ct_req = qla2x00_prep_ct_req(ha->ct_sns, GID_PT_CMD, gid_pt_rsp_size);
struct ct_sns_req *ct_req;
struct ct_sns_rsp *ct_rsp;
struct qla_hw_data *ha = vha->hw;
+ struct ct_arg arg;
if (IS_QLA2100(ha) || IS_QLA2200(ha))
return qla2x00_sns_gpn_id(vha, list);
+ arg.iocb = ha->ms_iocb;
+ arg.req_dma = ha->ct_sns_dma;
+ arg.rsp_dma = ha->ct_sns_dma;
+ arg.req_size = GPN_ID_REQ_SIZE;
+ arg.rsp_size = GPN_ID_RSP_SIZE;
+ arg.nport_handle = NPH_SNS;
+
for (i = 0; i < ha->max_fibre_devices; i++) {
/* Issue GPN_ID */
/* Prepare common MS IOCB */
- ms_pkt = ha->isp_ops->prep_ms_iocb(vha, GPN_ID_REQ_SIZE,
- GPN_ID_RSP_SIZE);
+ ms_pkt = ha->isp_ops->prep_ms_iocb(vha, &arg);
/* Prepare CT request */
ct_req = qla2x00_prep_ct_req(ha->ct_sns, GPN_ID_CMD,
ms_iocb_entry_t *ms_pkt;
struct ct_sns_req *ct_req;
struct ct_sns_rsp *ct_rsp;
+ struct ct_arg arg;
if (IS_QLA2100(ha) || IS_QLA2200(ha))
return qla2x00_sns_gnn_id(vha, list);
+ arg.iocb = ha->ms_iocb;
+ arg.req_dma = ha->ct_sns_dma;
+ arg.rsp_dma = ha->ct_sns_dma;
+ arg.req_size = GNN_ID_REQ_SIZE;
+ arg.rsp_size = GNN_ID_RSP_SIZE;
+ arg.nport_handle = NPH_SNS;
+
for (i = 0; i < ha->max_fibre_devices; i++) {
/* Issue GNN_ID */
/* Prepare common MS IOCB */
- ms_pkt = ha->isp_ops->prep_ms_iocb(vha, GNN_ID_REQ_SIZE,
- GNN_ID_RSP_SIZE);
+ ms_pkt = ha->isp_ops->prep_ms_iocb(vha, &arg);
/* Prepare CT request */
ct_req = qla2x00_prep_ct_req(ha->ct_sns, GNN_ID_CMD,
ms_iocb_entry_t *ms_pkt;
struct ct_sns_req *ct_req;
struct ct_sns_rsp *ct_rsp;
+ struct ct_arg arg;
if (IS_QLA2100(ha) || IS_QLA2200(ha))
return qla2x00_sns_rft_id(vha);
+ arg.iocb = ha->ms_iocb;
+ arg.req_dma = ha->ct_sns_dma;
+ arg.rsp_dma = ha->ct_sns_dma;
+ arg.req_size = RFT_ID_REQ_SIZE;
+ arg.rsp_size = RFT_ID_RSP_SIZE;
+ arg.nport_handle = NPH_SNS;
+
/* Issue RFT_ID */
/* Prepare common MS IOCB */
- ms_pkt = ha->isp_ops->prep_ms_iocb(vha, RFT_ID_REQ_SIZE,
- RFT_ID_RSP_SIZE);
+ ms_pkt = ha->isp_ops->prep_ms_iocb(vha, &arg);
/* Prepare CT request */
ct_req = qla2x00_prep_ct_req(ha->ct_sns, RFT_ID_CMD,
ms_iocb_entry_t *ms_pkt;
struct ct_sns_req *ct_req;
struct ct_sns_rsp *ct_rsp;
+ struct ct_arg arg;
if (IS_QLA2100(ha) || IS_QLA2200(ha)) {
ql_dbg(ql_dbg_disc, vha, 0x2046,
return (QLA_SUCCESS);
}
+ arg.iocb = ha->ms_iocb;
+ arg.req_dma = ha->ct_sns_dma;
+ arg.rsp_dma = ha->ct_sns_dma;
+ arg.req_size = RFF_ID_REQ_SIZE;
+ arg.rsp_size = RFF_ID_RSP_SIZE;
+ arg.nport_handle = NPH_SNS;
+
/* Issue RFF_ID */
/* Prepare common MS IOCB */
- ms_pkt = ha->isp_ops->prep_ms_iocb(vha, RFF_ID_REQ_SIZE,
- RFF_ID_RSP_SIZE);
+ ms_pkt = ha->isp_ops->prep_ms_iocb(vha, &arg);
/* Prepare CT request */
ct_req = qla2x00_prep_ct_req(ha->ct_sns, RFF_ID_CMD,
ms_iocb_entry_t *ms_pkt;
struct ct_sns_req *ct_req;
struct ct_sns_rsp *ct_rsp;
+ struct ct_arg arg;
if (IS_QLA2100(ha) || IS_QLA2200(ha))
return qla2x00_sns_rnn_id(vha);
+ arg.iocb = ha->ms_iocb;
+ arg.req_dma = ha->ct_sns_dma;
+ arg.rsp_dma = ha->ct_sns_dma;
+ arg.req_size = RNN_ID_REQ_SIZE;
+ arg.rsp_size = RNN_ID_RSP_SIZE;
+ arg.nport_handle = NPH_SNS;
+
/* Issue RNN_ID */
/* Prepare common MS IOCB */
- ms_pkt = ha->isp_ops->prep_ms_iocb(vha, RNN_ID_REQ_SIZE,
- RNN_ID_RSP_SIZE);
+ ms_pkt = ha->isp_ops->prep_ms_iocb(vha, &arg);
/* Prepare CT request */
ct_req = qla2x00_prep_ct_req(ha->ct_sns, RNN_ID_CMD, RNN_ID_RSP_SIZE);
ms_iocb_entry_t *ms_pkt;
struct ct_sns_req *ct_req;
struct ct_sns_rsp *ct_rsp;
+ struct ct_arg arg;
if (IS_QLA2100(ha) || IS_QLA2200(ha)) {
ql_dbg(ql_dbg_disc, vha, 0x2050,
return (QLA_SUCCESS);
}
+ arg.iocb = ha->ms_iocb;
+ arg.req_dma = ha->ct_sns_dma;
+ arg.rsp_dma = ha->ct_sns_dma;
+ arg.req_size = 0;
+ arg.rsp_size = RSNN_NN_RSP_SIZE;
+ arg.nport_handle = NPH_SNS;
+
/* Issue RSNN_NN */
/* Prepare common MS IOCB */
/* Request size adjusted after CT preparation */
- ms_pkt = ha->isp_ops->prep_ms_iocb(vha, 0, RSNN_NN_RSP_SIZE);
+ ms_pkt = ha->isp_ops->prep_ms_iocb(vha, &arg);
/* Prepare CT request */
ct_req = qla2x00_prep_ct_req(ha->ct_sns, RSNN_NN_CMD,
*
* Returns 0 on success.
*/
-static int
+int
qla2x00_mgmt_svr_login(scsi_qla_host_t *vha)
{
int ret, rval;
ms_iocb_entry_t *ms_pkt;
struct ct_sns_req *ct_req;
struct ct_sns_rsp *ct_rsp;
+ struct ct_arg arg;
if (!IS_IIDMA_CAPABLE(ha))
return QLA_FUNCTION_FAILED;
+ arg.iocb = ha->ms_iocb;
+ arg.req_dma = ha->ct_sns_dma;
+ arg.rsp_dma = ha->ct_sns_dma;
+ arg.req_size = GFPN_ID_REQ_SIZE;
+ arg.rsp_size = GFPN_ID_RSP_SIZE;
+ arg.nport_handle = NPH_SNS;
+
for (i = 0; i < ha->max_fibre_devices; i++) {
/* Issue GFPN_ID */
/* Prepare common MS IOCB */
- ms_pkt = ha->isp_ops->prep_ms_iocb(vha, GFPN_ID_REQ_SIZE,
- GFPN_ID_RSP_SIZE);
+ ms_pkt = ha->isp_ops->prep_ms_iocb(vha, &arg);
/* Prepare CT request */
ct_req = qla2x00_prep_ct_req(ha->ct_sns, GFPN_ID_CMD,
return (rval);
}
-static inline void *
-qla24xx_prep_ms_fm_iocb(scsi_qla_host_t *vha, uint32_t req_size,
- uint32_t rsp_size)
-{
- struct ct_entry_24xx *ct_pkt;
- struct qla_hw_data *ha = vha->hw;
- ct_pkt = (struct ct_entry_24xx *)ha->ms_iocb;
- memset(ct_pkt, 0, sizeof(struct ct_entry_24xx));
-
- ct_pkt->entry_type = CT_IOCB_TYPE;
- ct_pkt->entry_count = 1;
- ct_pkt->nport_handle = cpu_to_le16(vha->mgmt_svr_loop_id);
- ct_pkt->timeout = cpu_to_le16(ha->r_a_tov / 10 * 2);
- ct_pkt->cmd_dsd_count = cpu_to_le16(1);
- ct_pkt->rsp_dsd_count = cpu_to_le16(1);
- ct_pkt->rsp_byte_count = cpu_to_le32(rsp_size);
- ct_pkt->cmd_byte_count = cpu_to_le32(req_size);
-
- ct_pkt->dseg_0_address[0] = cpu_to_le32(LSD(ha->ct_sns_dma));
- ct_pkt->dseg_0_address[1] = cpu_to_le32(MSD(ha->ct_sns_dma));
- ct_pkt->dseg_0_len = ct_pkt->cmd_byte_count;
-
- ct_pkt->dseg_1_address[0] = cpu_to_le32(LSD(ha->ct_sns_dma));
- ct_pkt->dseg_1_address[1] = cpu_to_le32(MSD(ha->ct_sns_dma));
- ct_pkt->dseg_1_len = ct_pkt->rsp_byte_count;
- ct_pkt->vp_index = vha->vp_idx;
-
- return ct_pkt;
-}
-
static inline struct ct_sns_req *
qla24xx_prep_ct_fm_req(struct ct_sns_pkt *p, uint16_t cmd,
int rval;
uint16_t i;
struct qla_hw_data *ha = vha->hw;
- ms_iocb_entry_t *ms_pkt;
+ ms_iocb_entry_t *ms_pkt;
struct ct_sns_req *ct_req;
struct ct_sns_rsp *ct_rsp;
+ struct ct_arg arg;
if (!IS_IIDMA_CAPABLE(ha))
return QLA_FUNCTION_FAILED;
if (rval)
return rval;
+ arg.iocb = ha->ms_iocb;
+ arg.req_dma = ha->ct_sns_dma;
+ arg.rsp_dma = ha->ct_sns_dma;
+ arg.req_size = GPSC_REQ_SIZE;
+ arg.rsp_size = GPSC_RSP_SIZE;
+ arg.nport_handle = vha->mgmt_svr_loop_id;
+
for (i = 0; i < ha->max_fibre_devices; i++) {
/* Issue GFPN_ID */
/* Prepare common MS IOCB */
- ms_pkt = qla24xx_prep_ms_fm_iocb(vha, GPSC_REQ_SIZE,
- GPSC_RSP_SIZE);
+ ms_pkt = qla24xx_prep_ms_iocb(vha, &arg);
/* Prepare CT request */
ct_req = qla24xx_prep_ct_fm_req(ha->ct_sns, GPSC_CMD,
struct ct_sns_rsp *ct_rsp;
struct qla_hw_data *ha = vha->hw;
uint8_t fcp_scsi_features = 0;
+ struct ct_arg arg;
for (i = 0; i < ha->max_fibre_devices; i++) {
/* Set default FC4 Type as UNKNOWN so the default is to
if (!IS_FWI2_CAPABLE(ha))
continue;
+ arg.iocb = ha->ms_iocb;
+ arg.req_dma = ha->ct_sns_dma;
+ arg.rsp_dma = ha->ct_sns_dma;
+ arg.req_size = GFF_ID_REQ_SIZE;
+ arg.rsp_size = GFF_ID_RSP_SIZE;
+ arg.nport_handle = NPH_SNS;
+
/* Prepare common MS IOCB */
- ms_pkt = ha->isp_ops->prep_ms_iocb(vha, GFF_ID_REQ_SIZE,
- GFF_ID_RSP_SIZE);
+ ms_pkt = ha->isp_ops->prep_ms_iocb(vha, &arg);
/* Prepare CT request */
ct_req = qla2x00_prep_ct_req(ha->ct_sns, GFF_ID_CMD,
break;
}
}
+
+/* GID_PN completion processing. */
+void qla24xx_handle_gidpn_event(scsi_qla_host_t *vha, struct event_arg *ea)
+{
+ fc_port_t *fcport = ea->fcport;
+
+ ql_dbg(ql_dbg_disc, vha, 0xffff,
+ "%s %8phC login state %d \n",
+ __func__, fcport->port_name, fcport->fw_login_state);
+
+ if (ea->sp->gen2 != fcport->login_gen) {
+ /* PLOGI/PRLI/LOGO came in while cmd was out.*/
+ ql_dbg(ql_dbg_disc, vha, 0xffff,
+ "%s %8phC generation changed rscn %d|%d login %d|%d \n",
+ __func__, fcport->port_name, fcport->last_rscn_gen,
+ fcport->rscn_gen, fcport->last_login_gen, fcport->login_gen);
+ return;
+ }
+
+ if (!ea->rc) {
+ if (ea->sp->gen1 == fcport->rscn_gen) {
+ fcport->scan_state = QLA_FCPORT_FOUND;
+ fcport->flags |= FCF_FABRIC_DEVICE;
+
+ if (fcport->d_id.b24 == ea->id.b24) {
+ /* cable plugged into the same place */
+ switch (vha->host->active_mode) {
+ case MODE_TARGET:
+ /* NOOP. let the other guy login to us.*/
+ break;
+ case MODE_INITIATOR:
+ case MODE_DUAL:
+ default:
+ if (atomic_read(&fcport->state) ==
+ FCS_ONLINE)
+ break;
+ ql_dbg(ql_dbg_disc, vha, 0xffff,
+ "%s %d %8phC post gnl\n",
+ __func__, __LINE__, fcport->port_name);
+ qla24xx_post_gnl_work(vha, fcport);
+ break;
+ }
+ } else { /* fcport->d_id.b24 != ea->id.b24 */
+ fcport->d_id.b24 = ea->id.b24;
+ if (fcport->deleted == QLA_SESS_DELETED) {
+ ql_dbg(ql_dbg_disc, vha, 0xffff,
+ "%s %d %8phC post del sess\n",
+ __func__, __LINE__, fcport->port_name);
+ qlt_schedule_sess_for_deletion_lock(fcport);
+ }
+ }
+ } else { /* ea->sp->gen1 != fcport->rscn_gen */
+ ql_dbg(ql_dbg_disc, vha, 0xffff,
+ "%s %d %8phC post gidpn\n",
+ __func__, __LINE__, fcport->port_name);
+ /* rscn came in while cmd was out */
+ qla24xx_post_gidpn_work(vha, fcport);
+ }
+ } else { /* ea->rc */
+ /* cable pulled */
+ if (ea->sp->gen1 == fcport->rscn_gen) {
+ if (ea->sp->gen2 == fcport->login_gen) {
+ ql_dbg(ql_dbg_disc, vha, 0xffff,
+ "%s %d %8phC post del sess\n", __func__,
+ __LINE__, fcport->port_name);
+ qlt_schedule_sess_for_deletion_lock(fcport);
+ } else {
+ ql_dbg(ql_dbg_disc, vha, 0xffff,
+ "%s %d %8phC login\n", __func__, __LINE__,
+ fcport->port_name);
+ qla24xx_fcport_handle_login(vha, fcport);
+ }
+ } else {
+ ql_dbg(ql_dbg_disc, vha, 0xffff,
+ "%s %d %8phC post gidpn\n", __func__, __LINE__,
+ fcport->port_name);
+ qla24xx_post_gidpn_work(vha, fcport);
+ }
+ }
+} /* gidpn_event */
+
+static void qla2x00_async_gidpn_sp_done(void *s, int res)
+{
+ struct srb *sp = s;
+ struct scsi_qla_host *vha = sp->vha;
+ fc_port_t *fcport = sp->fcport;
+ u8 *id = fcport->ct_desc.ct_sns->p.rsp.rsp.gid_pn.port_id;
+ struct event_arg ea;
+
+ fcport->flags &= ~FCF_ASYNC_SENT;
+
+ memset(&ea, 0, sizeof(ea));
+ ea.fcport = fcport;
+ ea.id.b.domain = id[0];
+ ea.id.b.area = id[1];
+ ea.id.b.al_pa = id[2];
+ ea.sp = sp;
+ ea.rc = res;
+ ea.event = FCME_GIDPN_DONE;
+
+ ql_dbg(ql_dbg_disc, vha, 0xffff,
+ "Async done-%s res %x, WWPN %8phC ID %3phC \n",
+ sp->name, res, fcport->port_name, id);
+
+ qla2x00_fcport_event_handler(vha, &ea);
+
+ sp->free(sp);
+}
+
+int qla24xx_async_gidpn(scsi_qla_host_t *vha, fc_port_t *fcport)
+{
+ int rval = QLA_FUNCTION_FAILED;
+ struct ct_sns_req *ct_req;
+ srb_t *sp;
+
+ if (!vha->flags.online)
+ goto done;
+
+ fcport->flags |= FCF_ASYNC_SENT;
+ fcport->disc_state = DSC_GID_PN;
+ fcport->scan_state = QLA_FCPORT_SCAN;
+ sp = qla2x00_get_sp(vha, fcport, GFP_ATOMIC);
+ if (!sp)
+ goto done;
+
+ sp->type = SRB_CT_PTHRU_CMD;
+ sp->name = "gidpn";
+ sp->gen1 = fcport->rscn_gen;
+ sp->gen2 = fcport->login_gen;
+
+ qla2x00_init_timer(sp, qla2x00_get_async_timeout(vha) + 2);
+
+ /* CT_IU preamble */
+ ct_req = qla2x00_prep_ct_req(fcport->ct_desc.ct_sns, GID_PN_CMD,
+ GID_PN_RSP_SIZE);
+
+ /* GIDPN req */
+ memcpy(ct_req->req.gid_pn.port_name, fcport->port_name,
+ WWN_SIZE);
+
+ /* req & rsp use the same buffer */
+ sp->u.iocb_cmd.u.ctarg.req = fcport->ct_desc.ct_sns;
+ sp->u.iocb_cmd.u.ctarg.req_dma = fcport->ct_desc.ct_sns_dma;
+ sp->u.iocb_cmd.u.ctarg.rsp = fcport->ct_desc.ct_sns;
+ sp->u.iocb_cmd.u.ctarg.rsp_dma = fcport->ct_desc.ct_sns_dma;
+ sp->u.iocb_cmd.u.ctarg.req_size = GID_PN_REQ_SIZE;
+ sp->u.iocb_cmd.u.ctarg.rsp_size = GID_PN_RSP_SIZE;
+ sp->u.iocb_cmd.u.ctarg.nport_handle = NPH_SNS;
+
+ sp->u.iocb_cmd.timeout = qla2x00_async_iocb_timeout;
+ sp->done = qla2x00_async_gidpn_sp_done;
+
+ rval = qla2x00_start_sp(sp);
+ if (rval != QLA_SUCCESS)
+ goto done_free_sp;
+
+ ql_dbg(ql_dbg_disc, vha, 0x206f,
+ "Async-%s - %8phC hdl=%x loopid=%x portid %02x%02x%02x.\n",
+ sp->name, fcport->port_name,
+ sp->handle, fcport->loop_id, fcport->d_id.b.domain,
+ fcport->d_id.b.area, fcport->d_id.b.al_pa);
+ return rval;
+
+done_free_sp:
+ sp->free(sp);
+done:
+ fcport->flags &= ~FCF_ASYNC_SENT;
+ return rval;
+}
+
+int qla24xx_post_gidpn_work(struct scsi_qla_host *vha, fc_port_t *fcport)
+{
+ struct qla_work_evt *e;
+ int ls;
+
+ ls = atomic_read(&vha->loop_state);
+ if (((ls != LOOP_READY) && (ls != LOOP_UP)) ||
+ test_bit(UNLOADING, &vha->dpc_flags))
+ return 0;
+
+ e = qla2x00_alloc_work(vha, QLA_EVT_GIDPN);
+ if (!e)
+ return QLA_FUNCTION_FAILED;
+
+ e->u.fcport.fcport = fcport;
+ return qla2x00_post_work(vha, e);
+}
+
+int qla24xx_post_gpsc_work(struct scsi_qla_host *vha, fc_port_t *fcport)
+{
+ struct qla_work_evt *e;
+
+ e = qla2x00_alloc_work(vha, QLA_EVT_GPSC);
+ if (!e)
+ return QLA_FUNCTION_FAILED;
+
+ e->u.fcport.fcport = fcport;
+ return qla2x00_post_work(vha, e);
+}
+
+static void qla24xx_async_gpsc_sp_done(void *s, int res)
+{
+ struct srb *sp = s;
+ struct scsi_qla_host *vha = sp->vha;
+ struct qla_hw_data *ha = vha->hw;
+ fc_port_t *fcport = sp->fcport;
+ struct ct_sns_rsp *ct_rsp;
+ struct event_arg ea;
+
+ ct_rsp = &fcport->ct_desc.ct_sns->p.rsp;
+
+ ql_dbg(ql_dbg_disc, vha, 0xffff,
+ "Async done-%s res %x, WWPN %8phC \n",
+ sp->name, res, fcport->port_name);
+
+ fcport->flags &= ~FCF_ASYNC_SENT;
+
+ if (res == (DID_ERROR << 16)) {
+ /* entry status error */
+ goto done;
+ } else if (res) {
+ if ((ct_rsp->header.reason_code ==
+ CT_REASON_INVALID_COMMAND_CODE) ||
+ (ct_rsp->header.reason_code ==
+ CT_REASON_COMMAND_UNSUPPORTED)) {
+ ql_dbg(ql_dbg_disc, vha, 0x205a,
+ "GPSC command unsupported, disabling "
+ "query.\n");
+ ha->flags.gpsc_supported = 0;
+ res = QLA_SUCCESS;
+ }
+ } else {
+ switch (be16_to_cpu(ct_rsp->rsp.gpsc.speed)) {
+ case BIT_15:
+ fcport->fp_speed = PORT_SPEED_1GB;
+ break;
+ case BIT_14:
+ fcport->fp_speed = PORT_SPEED_2GB;
+ break;
+ case BIT_13:
+ fcport->fp_speed = PORT_SPEED_4GB;
+ break;
+ case BIT_12:
+ fcport->fp_speed = PORT_SPEED_10GB;
+ break;
+ case BIT_11:
+ fcport->fp_speed = PORT_SPEED_8GB;
+ break;
+ case BIT_10:
+ fcport->fp_speed = PORT_SPEED_16GB;
+ break;
+ case BIT_8:
+ fcport->fp_speed = PORT_SPEED_32GB;
+ break;
+ }
+
+ ql_dbg(ql_dbg_disc, vha, 0xffff,
+ "Async-%s OUT WWPN %8phC speeds=%04x speed=%04x.\n",
+ sp->name,
+ fcport->fabric_port_name,
+ be16_to_cpu(ct_rsp->rsp.gpsc.speeds),
+ be16_to_cpu(ct_rsp->rsp.gpsc.speed));
+ }
+done:
+ memset(&ea, 0, sizeof(ea));
+ ea.event = FCME_GPSC_DONE;
+ ea.rc = res;
+ ea.fcport = fcport;
+ qla2x00_fcport_event_handler(vha, &ea);
+
+ sp->free(sp);
+}
+
+int qla24xx_async_gpsc(scsi_qla_host_t *vha, fc_port_t *fcport)
+{
+ int rval = QLA_FUNCTION_FAILED;
+ struct ct_sns_req *ct_req;
+ srb_t *sp;
+
+ if (!vha->flags.online)
+ goto done;
+
+ fcport->flags |= FCF_ASYNC_SENT;
+ sp = qla2x00_get_sp(vha, fcport, GFP_KERNEL);
+ if (!sp)
+ goto done;
+
+ sp->type = SRB_CT_PTHRU_CMD;
+ sp->name = "gpsc";
+ sp->gen1 = fcport->rscn_gen;
+ sp->gen2 = fcport->login_gen;
+
+ qla2x00_init_timer(sp, qla2x00_get_async_timeout(vha) + 2);
+
+ /* CT_IU preamble */
+ ct_req = qla24xx_prep_ct_fm_req(fcport->ct_desc.ct_sns, GPSC_CMD,
+ GPSC_RSP_SIZE);
+
+ /* GPSC req */
+ memcpy(ct_req->req.gpsc.port_name, fcport->port_name,
+ WWN_SIZE);
+
+ sp->u.iocb_cmd.u.ctarg.req = fcport->ct_desc.ct_sns;
+ sp->u.iocb_cmd.u.ctarg.req_dma = fcport->ct_desc.ct_sns_dma;
+ sp->u.iocb_cmd.u.ctarg.rsp = fcport->ct_desc.ct_sns;
+ sp->u.iocb_cmd.u.ctarg.rsp_dma = fcport->ct_desc.ct_sns_dma;
+ sp->u.iocb_cmd.u.ctarg.req_size = GPSC_REQ_SIZE;
+ sp->u.iocb_cmd.u.ctarg.rsp_size = GPSC_RSP_SIZE;
+ sp->u.iocb_cmd.u.ctarg.nport_handle = vha->mgmt_svr_loop_id;
+
+ sp->u.iocb_cmd.timeout = qla2x00_async_iocb_timeout;
+ sp->done = qla24xx_async_gpsc_sp_done;
+
+ rval = qla2x00_start_sp(sp);
+ if (rval != QLA_SUCCESS)
+ goto done_free_sp;
+
+ ql_dbg(ql_dbg_disc, vha, 0xffff,
+ "Async-%s %8phC hdl=%x loopid=%x portid=%02x%02x%02x.\n",
+ sp->name, fcport->port_name, sp->handle,
+ fcport->loop_id, fcport->d_id.b.domain,
+ fcport->d_id.b.area, fcport->d_id.b.al_pa);
+ return rval;
+
+done_free_sp:
+ sp->free(sp);
+done:
+ fcport->flags &= ~FCF_ASYNC_SENT;
+ return rval;
+}
+
+int qla24xx_post_gpnid_work(struct scsi_qla_host *vha, port_id_t *id)
+{
+ struct qla_work_evt *e;
+
+ if (test_bit(UNLOADING, &vha->dpc_flags))
+ return 0;
+
+ e = qla2x00_alloc_work(vha, QLA_EVT_GPNID);
+ if (!e)
+ return QLA_FUNCTION_FAILED;
+
+ e->u.gpnid.id = *id;
+ return qla2x00_post_work(vha, e);
+}
+
+void qla24xx_async_gpnid_done(scsi_qla_host_t *vha, srb_t *sp)
+{
+ if (sp->u.iocb_cmd.u.ctarg.req) {
+ dma_free_coherent(&vha->hw->pdev->dev,
+ sizeof(struct ct_sns_pkt),
+ sp->u.iocb_cmd.u.ctarg.req,
+ sp->u.iocb_cmd.u.ctarg.req_dma);
+ sp->u.iocb_cmd.u.ctarg.req = NULL;
+ }
+ if (sp->u.iocb_cmd.u.ctarg.rsp) {
+ dma_free_coherent(&vha->hw->pdev->dev,
+ sizeof(struct ct_sns_pkt),
+ sp->u.iocb_cmd.u.ctarg.rsp,
+ sp->u.iocb_cmd.u.ctarg.rsp_dma);
+ sp->u.iocb_cmd.u.ctarg.rsp = NULL;
+ }
+
+ sp->free(sp);
+}
+
+void qla24xx_handle_gpnid_event(scsi_qla_host_t *vha, struct event_arg *ea)
+{
+ fc_port_t *fcport;
+ unsigned long flags;
+
+ spin_lock_irqsave(&vha->hw->tgt.sess_lock, flags);
+ fcport = qla2x00_find_fcport_by_wwpn(vha, ea->port_name, 1);
+ spin_unlock_irqrestore(&vha->hw->tgt.sess_lock, flags);
+
+ if (fcport) {
+ /* cable moved. just plugged in */
+ ql_dbg(ql_dbg_disc, vha, 0xffff,
+ "%s %d %8phC post del sess\n",
+ __func__, __LINE__, fcport->port_name);
+
+ fcport->rscn_gen++;
+ fcport->d_id = ea->id;
+ fcport->scan_state = QLA_FCPORT_FOUND;
+ fcport->flags |= FCF_FABRIC_DEVICE;
+
+ qlt_schedule_sess_for_deletion_lock(fcport);
+ } else {
+ /* create new fcport */
+ ql_dbg(ql_dbg_disc, vha, 0xffff,
+ "%s %d %8phC post new sess\n",
+ __func__, __LINE__, ea->port_name);
+
+ qla24xx_post_newsess_work(vha, &ea->id, ea->port_name, NULL);
+ }
+}
+
+static void qla2x00_async_gpnid_sp_done(void *s, int res)
+{
+ struct srb *sp = s;
+ struct scsi_qla_host *vha = sp->vha;
+ struct ct_sns_req *ct_req =
+ (struct ct_sns_req *)sp->u.iocb_cmd.u.ctarg.req;
+ struct ct_sns_rsp *ct_rsp =
+ (struct ct_sns_rsp *)sp->u.iocb_cmd.u.ctarg.rsp;
+ struct event_arg ea;
+ struct qla_work_evt *e;
+
+ ql_dbg(ql_dbg_disc, vha, 0xffff,
+ "Async done-%s res %x ID %3phC. %8phC\n",
+ sp->name, res, ct_req->req.port_id.port_id,
+ ct_rsp->rsp.gpn_id.port_name);
+
+ memset(&ea, 0, sizeof(ea));
+ memcpy(ea.port_name, ct_rsp->rsp.gpn_id.port_name, WWN_SIZE);
+ ea.sp = sp;
+ ea.id.b.domain = ct_req->req.port_id.port_id[0];
+ ea.id.b.area = ct_req->req.port_id.port_id[1];
+ ea.id.b.al_pa = ct_req->req.port_id.port_id[2];
+ ea.rc = res;
+ ea.event = FCME_GPNID_DONE;
+
+ qla2x00_fcport_event_handler(vha, &ea);
+
+ e = qla2x00_alloc_work(vha, QLA_EVT_GPNID_DONE);
+ if (!e) {
+ /* please ignore kernel warning. otherwise, we have mem leak. */
+ if (sp->u.iocb_cmd.u.ctarg.req) {
+ dma_free_coherent(&vha->hw->pdev->dev,
+ sizeof(struct ct_sns_pkt),
+ sp->u.iocb_cmd.u.ctarg.req,
+ sp->u.iocb_cmd.u.ctarg.req_dma);
+ sp->u.iocb_cmd.u.ctarg.req = NULL;
+ }
+ if (sp->u.iocb_cmd.u.ctarg.rsp) {
+ dma_free_coherent(&vha->hw->pdev->dev,
+ sizeof(struct ct_sns_pkt),
+ sp->u.iocb_cmd.u.ctarg.rsp,
+ sp->u.iocb_cmd.u.ctarg.rsp_dma);
+ sp->u.iocb_cmd.u.ctarg.rsp = NULL;
+ }
+
+ sp->free(sp);
+ return;
+ }
+
+ e->u.iosb.sp = sp;
+ qla2x00_post_work(vha, e);
+}
+
+/* Get WWPN with Nport ID. */
+int qla24xx_async_gpnid(scsi_qla_host_t *vha, port_id_t *id)
+{
+ int rval = QLA_FUNCTION_FAILED;
+ struct ct_sns_req *ct_req;
+ srb_t *sp;
+ struct ct_sns_pkt *ct_sns;
+
+ if (!vha->flags.online)
+ goto done;
+
+ sp = qla2x00_get_sp(vha, NULL, GFP_KERNEL);
+ if (!sp)
+ goto done;
+
+ sp->type = SRB_CT_PTHRU_CMD;
+ sp->name = "gpnid";
+ qla2x00_init_timer(sp, qla2x00_get_async_timeout(vha) + 2);
+
+ sp->u.iocb_cmd.u.ctarg.req = dma_alloc_coherent(&vha->hw->pdev->dev,
+ sizeof(struct ct_sns_pkt), &sp->u.iocb_cmd.u.ctarg.req_dma,
+ GFP_KERNEL);
+ if (!sp->u.iocb_cmd.u.ctarg.req) {
+ ql_log(ql_log_warn, vha, 0xffff,
+ "Failed to allocate ct_sns request.\n");
+ goto done_free_sp;
+ }
+
+ sp->u.iocb_cmd.u.ctarg.rsp = dma_alloc_coherent(&vha->hw->pdev->dev,
+ sizeof(struct ct_sns_pkt), &sp->u.iocb_cmd.u.ctarg.rsp_dma,
+ GFP_KERNEL);
+ if (!sp->u.iocb_cmd.u.ctarg.rsp) {
+ ql_log(ql_log_warn, vha, 0xffff,
+ "Failed to allocate ct_sns request.\n");
+ goto done_free_sp;
+ }
+
+ ct_sns = (struct ct_sns_pkt *)sp->u.iocb_cmd.u.ctarg.rsp;
+ memset(ct_sns, 0, sizeof(*ct_sns));
+
+ ct_sns = (struct ct_sns_pkt *)sp->u.iocb_cmd.u.ctarg.req;
+ /* CT_IU preamble */
+ ct_req = qla2x00_prep_ct_req(ct_sns, GPN_ID_CMD, GPN_ID_RSP_SIZE);
+
+ /* GPN_ID req */
+ ct_req->req.port_id.port_id[0] = id->b.domain;
+ ct_req->req.port_id.port_id[1] = id->b.area;
+ ct_req->req.port_id.port_id[2] = id->b.al_pa;
+
+ sp->u.iocb_cmd.u.ctarg.req_size = GPN_ID_REQ_SIZE;
+ sp->u.iocb_cmd.u.ctarg.rsp_size = GPN_ID_RSP_SIZE;
+ sp->u.iocb_cmd.u.ctarg.nport_handle = NPH_SNS;
+
+ sp->u.iocb_cmd.timeout = qla2x00_async_iocb_timeout;
+ sp->done = qla2x00_async_gpnid_sp_done;
+
+ rval = qla2x00_start_sp(sp);
+ if (rval != QLA_SUCCESS)
+ goto done_free_sp;
+
+ ql_dbg(ql_dbg_disc, vha, 0xffff,
+ "Async-%s hdl=%x ID %3phC.\n", sp->name,
+ sp->handle, ct_req->req.port_id.port_id);
+ return rval;
+
+done_free_sp:
+ if (sp->u.iocb_cmd.u.ctarg.req) {
+ dma_free_coherent(&vha->hw->pdev->dev,
+ sizeof(struct ct_sns_pkt),
+ sp->u.iocb_cmd.u.ctarg.req,
+ sp->u.iocb_cmd.u.ctarg.req_dma);
+ sp->u.iocb_cmd.u.ctarg.req = NULL;
+ }
+ if (sp->u.iocb_cmd.u.ctarg.rsp) {
+ dma_free_coherent(&vha->hw->pdev->dev,
+ sizeof(struct ct_sns_pkt),
+ sp->u.iocb_cmd.u.ctarg.rsp,
+ sp->u.iocb_cmd.u.ctarg.rsp_dma);
+ sp->u.iocb_cmd.u.ctarg.rsp = NULL;
+ }
+
+ sp->free(sp);
+done:
+ return rval;
+}
static int qla2x00_configure_loop(scsi_qla_host_t *);
static int qla2x00_configure_local_loop(scsi_qla_host_t *);
static int qla2x00_configure_fabric(scsi_qla_host_t *);
-static int qla2x00_find_all_fabric_devs(scsi_qla_host_t *, struct list_head *);
-static int qla2x00_fabric_dev_login(scsi_qla_host_t *, fc_port_t *,
- uint16_t *);
-
+static int qla2x00_find_all_fabric_devs(scsi_qla_host_t *);
static int qla2x00_restart_isp(scsi_qla_host_t *);
static struct qla_chip_state_84xx *qla84xx_get_chip(struct scsi_qla_host *);
static int qla84xx_init_chip(scsi_qla_host_t *);
static int qla25xx_init_queues(struct qla_hw_data *);
+static int qla24xx_post_gpdb_work(struct scsi_qla_host *, fc_port_t *, u8);
+static void qla24xx_handle_plogi_done_event(struct scsi_qla_host *,
+ struct event_arg *);
/* SRB Extensions ---------------------------------------------------------- */
{
srb_t *sp = (srb_t *)__data;
struct srb_iocb *iocb;
- fc_port_t *fcport = sp->fcport;
- struct qla_hw_data *ha = fcport->vha->hw;
+ scsi_qla_host_t *vha = sp->vha;
struct req_que *req;
unsigned long flags;
- spin_lock_irqsave(&ha->hardware_lock, flags);
- req = ha->req_q_map[0];
+ spin_lock_irqsave(&vha->hw->hardware_lock, flags);
+ req = vha->hw->req_q_map[0];
req->outstanding_cmds[sp->handle] = NULL;
iocb = &sp->u.iocb_cmd;
iocb->timeout(sp);
- sp->free(fcport->vha, sp);
- spin_unlock_irqrestore(&ha->hardware_lock, flags);
+ sp->free(sp);
+ spin_unlock_irqrestore(&vha->hw->hardware_lock, flags);
}
void
-qla2x00_sp_free(void *data, void *ptr)
+qla2x00_sp_free(void *ptr)
{
- srb_t *sp = (srb_t *)ptr;
+ srb_t *sp = ptr;
struct srb_iocb *iocb = &sp->u.iocb_cmd;
- struct scsi_qla_host *vha = (scsi_qla_host_t *)data;
del_timer(&iocb->timer);
- qla2x00_rel_sp(vha, sp);
+ qla2x00_rel_sp(sp);
}
/* Asynchronous Login/Logout Routines -------------------------------------- */
return tmo;
}
-static void
+void
qla2x00_async_iocb_timeout(void *data)
{
- srb_t *sp = (srb_t *)data;
+ srb_t *sp = data;
fc_port_t *fcport = sp->fcport;
+ struct srb_iocb *lio = &sp->u.iocb_cmd;
+ struct event_arg ea;
ql_dbg(ql_dbg_disc, fcport->vha, 0x2071,
- "Async-%s timeout - hdl=%x portid=%02x%02x%02x.\n",
- sp->name, sp->handle, fcport->d_id.b.domain, fcport->d_id.b.area,
- fcport->d_id.b.al_pa);
+ "Async-%s timeout - hdl=%x portid=%06x %8phC.\n",
+ sp->name, sp->handle, fcport->d_id.b24, fcport->port_name);
fcport->flags &= ~FCF_ASYNC_SENT;
- if (sp->type == SRB_LOGIN_CMD) {
- struct srb_iocb *lio = &sp->u.iocb_cmd;
- qla2x00_post_async_logout_work(fcport->vha, fcport, NULL);
+
+ switch (sp->type) {
+ case SRB_LOGIN_CMD:
/* Retry as needed. */
lio->u.logio.data[0] = MBS_COMMAND_ERROR;
lio->u.logio.data[1] = lio->u.logio.flags & SRB_LOGIN_RETRIED ?
QLA_LOGIO_LOGIN_RETRIED : 0;
- qla2x00_post_async_login_done_work(fcport->vha, fcport,
- lio->u.logio.data);
- } else if (sp->type == SRB_LOGOUT_CMD) {
+ memset(&ea, 0, sizeof(ea));
+ ea.event = FCME_PLOGI_DONE;
+ ea.fcport = sp->fcport;
+ ea.data[0] = lio->u.logio.data[0];
+ ea.data[1] = lio->u.logio.data[1];
+ ea.sp = sp;
+ qla24xx_handle_plogi_done_event(fcport->vha, &ea);
+ break;
+ case SRB_LOGOUT_CMD:
qlt_logo_completion_handler(fcport, QLA_FUNCTION_TIMEOUT);
+ break;
+ case SRB_CT_PTHRU_CMD:
+ case SRB_MB_IOCB:
+ case SRB_NACK_PLOGI:
+ case SRB_NACK_PRLI:
+ case SRB_NACK_LOGO:
+ sp->done(sp, QLA_FUNCTION_TIMEOUT);
+ break;
}
}
static void
-qla2x00_async_login_sp_done(void *data, void *ptr, int res)
+qla2x00_async_login_sp_done(void *ptr, int res)
{
- srb_t *sp = (srb_t *)ptr;
+ srb_t *sp = ptr;
+ struct scsi_qla_host *vha = sp->vha;
struct srb_iocb *lio = &sp->u.iocb_cmd;
- struct scsi_qla_host *vha = (scsi_qla_host_t *)data;
+ struct event_arg ea;
- if (!test_bit(UNLOADING, &vha->dpc_flags))
- qla2x00_post_async_login_done_work(sp->fcport->vha, sp->fcport,
- lio->u.logio.data);
- sp->free(sp->fcport->vha, sp);
+ ql_dbg(ql_dbg_disc, vha, 0xffff,
+ "%s %8phC res %d \n", __func__, sp->fcport->port_name, res);
+
+ sp->fcport->flags &= ~FCF_ASYNC_SENT;
+ if (!test_bit(UNLOADING, &vha->dpc_flags)) {
+ memset(&ea, 0, sizeof(ea));
+ ea.event = FCME_PLOGI_DONE;
+ ea.fcport = sp->fcport;
+ ea.data[0] = lio->u.logio.data[0];
+ ea.data[1] = lio->u.logio.data[1];
+ ea.iop[0] = lio->u.logio.iop[0];
+ ea.iop[1] = lio->u.logio.iop[1];
+ ea.sp = sp;
+ qla2x00_fcport_event_handler(vha, &ea);
+ }
+
+ sp->free(sp);
}
int
{
srb_t *sp;
struct srb_iocb *lio;
- int rval;
+ int rval = QLA_FUNCTION_FAILED;
+
+ if (!vha->flags.online)
+ goto done;
+
+ if ((fcport->fw_login_state == DSC_LS_PLOGI_PEND) ||
+ (fcport->fw_login_state == DSC_LS_PLOGI_COMP) ||
+ (fcport->fw_login_state == DSC_LS_PRLI_PEND))
+ goto done;
- rval = QLA_FUNCTION_FAILED;
sp = qla2x00_get_sp(vha, fcport, GFP_KERNEL);
if (!sp)
goto done;
+ fcport->flags |= FCF_ASYNC_SENT;
+ fcport->logout_completed = 0;
+
sp->type = SRB_LOGIN_CMD;
sp->name = "login";
qla2x00_init_timer(sp, qla2x00_get_async_timeout(vha) + 2);
}
ql_dbg(ql_dbg_disc, vha, 0x2072,
- "Async-login - hdl=%x, loopid=%x portid=%02x%02x%02x "
- "retries=%d.\n", sp->handle, fcport->loop_id,
+ "Async-login - %8phC hdl=%x, loopid=%x portid=%02x%02x%02x "
+ "retries=%d.\n", fcport->port_name, sp->handle, fcport->loop_id,
fcport->d_id.b.domain, fcport->d_id.b.area, fcport->d_id.b.al_pa,
fcport->login_retry);
return rval;
done_free_sp:
- sp->free(fcport->vha, sp);
+ sp->free(sp);
done:
+ fcport->flags &= ~FCF_ASYNC_SENT;
return rval;
}
static void
-qla2x00_async_logout_sp_done(void *data, void *ptr, int res)
+qla2x00_async_logout_sp_done(void *ptr, int res)
{
- srb_t *sp = (srb_t *)ptr;
+ srb_t *sp = ptr;
struct srb_iocb *lio = &sp->u.iocb_cmd;
- struct scsi_qla_host *vha = (scsi_qla_host_t *)data;
- if (!test_bit(UNLOADING, &vha->dpc_flags))
- qla2x00_post_async_logout_done_work(sp->fcport->vha, sp->fcport,
+ sp->fcport->flags &= ~FCF_ASYNC_SENT;
+ if (!test_bit(UNLOADING, &sp->vha->dpc_flags))
+ qla2x00_post_async_logout_done_work(sp->vha, sp->fcport,
lio->u.logio.data);
- sp->free(sp->fcport->vha, sp);
+ sp->free(sp);
}
int
int rval;
rval = QLA_FUNCTION_FAILED;
+ fcport->flags |= FCF_ASYNC_SENT;
sp = qla2x00_get_sp(vha, fcport, GFP_KERNEL);
if (!sp)
goto done;
goto done_free_sp;
ql_dbg(ql_dbg_disc, vha, 0x2070,
- "Async-logout - hdl=%x loop-id=%x portid=%02x%02x%02x.\n",
+ "Async-logout - hdl=%x loop-id=%x portid=%02x%02x%02x %8phC.\n",
sp->handle, fcport->loop_id, fcport->d_id.b.domain,
- fcport->d_id.b.area, fcport->d_id.b.al_pa);
+ fcport->d_id.b.area, fcport->d_id.b.al_pa,
+ fcport->port_name);
return rval;
done_free_sp:
- sp->free(fcport->vha, sp);
+ sp->free(sp);
done:
+ fcport->flags &= ~FCF_ASYNC_SENT;
return rval;
}
static void
-qla2x00_async_adisc_sp_done(void *data, void *ptr, int res)
+qla2x00_async_adisc_sp_done(void *ptr, int res)
{
- srb_t *sp = (srb_t *)ptr;
+ srb_t *sp = ptr;
+ struct scsi_qla_host *vha = sp->vha;
struct srb_iocb *lio = &sp->u.iocb_cmd;
- struct scsi_qla_host *vha = (scsi_qla_host_t *)data;
if (!test_bit(UNLOADING, &vha->dpc_flags))
- qla2x00_post_async_adisc_done_work(sp->fcport->vha, sp->fcport,
+ qla2x00_post_async_adisc_done_work(sp->vha, sp->fcport,
lio->u.logio.data);
- sp->free(sp->fcport->vha, sp);
+ sp->free(sp);
}
int
int rval;
rval = QLA_FUNCTION_FAILED;
+ fcport->flags |= FCF_ASYNC_SENT;
sp = qla2x00_get_sp(vha, fcport, GFP_KERNEL);
if (!sp)
goto done;
if (rval != QLA_SUCCESS)
goto done_free_sp;
- ql_dbg(ql_dbg_disc, vha, 0x206f,
- "Async-adisc - hdl=%x loopid=%x portid=%02x%02x%02x.\n",
- sp->handle, fcport->loop_id, fcport->d_id.b.domain,
- fcport->d_id.b.area, fcport->d_id.b.al_pa);
- return rval;
+ ql_dbg(ql_dbg_disc, vha, 0x206f,
+ "Async-adisc - hdl=%x loopid=%x portid=%02x%02x%02x.\n",
+ sp->handle, fcport->loop_id, fcport->d_id.b.domain,
+ fcport->d_id.b.area, fcport->d_id.b.al_pa);
+ return rval;
+
+done_free_sp:
+ sp->free(sp);
+done:
+ fcport->flags &= ~FCF_ASYNC_SENT;
+ return rval;
+}
+
+static void qla24xx_handle_gnl_done_event(scsi_qla_host_t *vha,
+ struct event_arg *ea)
+{
+ fc_port_t *fcport, *conflict_fcport;
+ struct get_name_list_extended *e;
+ u16 i, n, found = 0, loop_id;
+ port_id_t id;
+ u64 wwn;
+ u8 opt = 0;
+
+ fcport = ea->fcport;
+
+ if (ea->rc) { /* rval */
+ if (fcport->login_retry == 0) {
+ fcport->login_retry = vha->hw->login_retry_count;
+ ql_dbg(ql_dbg_disc, vha, 0xffff,
+ "GNL failed Port login retry %8phN, retry cnt=%d.\n",
+ fcport->port_name, fcport->login_retry);
+ }
+ return;
+ }
+
+ if (fcport->last_rscn_gen != fcport->rscn_gen) {
+ ql_dbg(ql_dbg_disc, vha, 0xffff,
+ "%s %8phC rscn gen changed rscn %d|%d \n",
+ __func__, fcport->port_name,
+ fcport->last_rscn_gen, fcport->rscn_gen);
+ qla24xx_post_gidpn_work(vha, fcport);
+ return;
+ } else if (fcport->last_login_gen != fcport->login_gen) {
+ ql_dbg(ql_dbg_disc, vha, 0xffff,
+ "%s %8phC login gen changed login %d|%d \n",
+ __func__, fcport->port_name,
+ fcport->last_login_gen, fcport->login_gen);
+ return;
+ }
+
+ n = ea->data[0] / sizeof(struct get_name_list_extended);
+
+ ql_dbg(ql_dbg_disc, vha, 0xffff,
+ "%s %d %8phC n %d %02x%02x%02x lid %d \n",
+ __func__, __LINE__, fcport->port_name, n,
+ fcport->d_id.b.domain, fcport->d_id.b.area,
+ fcport->d_id.b.al_pa, fcport->loop_id);
+
+ for (i = 0; i < n; i++) {
+ e = &vha->gnl.l[i];
+ wwn = wwn_to_u64(e->port_name);
+
+ if (memcmp((u8 *)&wwn, fcport->port_name, WWN_SIZE))
+ continue;
+
+ found = 1;
+ id.b.domain = e->port_id[2];
+ id.b.area = e->port_id[1];
+ id.b.al_pa = e->port_id[0];
+ id.b.rsvd_1 = 0;
+
+ loop_id = le16_to_cpu(e->nport_handle);
+ loop_id = (loop_id & 0x7fff);
+
+ ql_dbg(ql_dbg_disc, vha, 0xffff,
+ "%s found %8phC CLS [%d|%d] ID[%02x%02x%02x|%02x%02x%02x] lid[%d|%d]\n",
+ __func__, fcport->port_name,
+ e->current_login_state, fcport->fw_login_state,
+ id.b.domain, id.b.area, id.b.al_pa,
+ fcport->d_id.b.domain, fcport->d_id.b.area,
+ fcport->d_id.b.al_pa, loop_id, fcport->loop_id);
+
+ if ((id.b24 != fcport->d_id.b24) ||
+ ((fcport->loop_id != FC_NO_LOOP_ID) &&
+ (fcport->loop_id != loop_id))) {
+ ql_dbg(ql_dbg_disc, vha, 0xffff,
+ "%s %d %8phC post del sess\n",
+ __func__, __LINE__, fcport->port_name);
+ qlt_schedule_sess_for_deletion(fcport, 1);
+ return;
+ }
+
+ fcport->loop_id = loop_id;
+
+ wwn = wwn_to_u64(fcport->port_name);
+ qlt_find_sess_invalidate_other(vha, wwn,
+ id, loop_id, &conflict_fcport);
+
+ if (conflict_fcport) {
+ /*
+ * Another share fcport share the same loop_id &
+ * nport id. Conflict fcport needs to finish
+ * cleanup before this fcport can proceed to login.
+ */
+ conflict_fcport->conflict = fcport;
+ fcport->login_pause = 1;
+ }
+
+ switch (e->current_login_state) {
+ case DSC_LS_PRLI_COMP:
+ ql_dbg(ql_dbg_disc, vha, 0xffff,
+ "%s %d %8phC post gpdb\n",
+ __func__, __LINE__, fcport->port_name);
+ opt = PDO_FORCE_ADISC;
+ qla24xx_post_gpdb_work(vha, fcport, opt);
+ break;
+
+ case DSC_LS_PORT_UNAVAIL:
+ default:
+ if (fcport->loop_id == FC_NO_LOOP_ID) {
+ qla2x00_find_new_loop_id(vha, fcport);
+ fcport->fw_login_state = DSC_LS_PORT_UNAVAIL;
+ }
+ ql_dbg(ql_dbg_disc, vha, 0xffff,
+ "%s %d %8phC \n",
+ __func__, __LINE__, fcport->port_name);
+ qla24xx_fcport_handle_login(vha, fcport);
+ break;
+ }
+ }
+
+ if (!found) {
+ /* fw has no record of this port */
+ if (fcport->loop_id == FC_NO_LOOP_ID) {
+ qla2x00_find_new_loop_id(vha, fcport);
+ fcport->fw_login_state = DSC_LS_PORT_UNAVAIL;
+ } else {
+ for (i = 0; i < n; i++) {
+ e = &vha->gnl.l[i];
+ id.b.domain = e->port_id[0];
+ id.b.area = e->port_id[1];
+ id.b.al_pa = e->port_id[2];
+ id.b.rsvd_1 = 0;
+ loop_id = le16_to_cpu(e->nport_handle);
+
+ if (fcport->d_id.b24 == id.b24) {
+ conflict_fcport =
+ qla2x00_find_fcport_by_wwpn(vha,
+ e->port_name, 0);
+
+ ql_dbg(ql_dbg_disc, vha, 0xffff,
+ "%s %d %8phC post del sess\n",
+ __func__, __LINE__,
+ conflict_fcport->port_name);
+ qlt_schedule_sess_for_deletion
+ (conflict_fcport, 1);
+ }
+
+ if (fcport->loop_id == loop_id) {
+ /* FW already picked this loop id for another fcport */
+ qla2x00_find_new_loop_id(vha, fcport);
+ }
+ }
+ }
+ qla24xx_fcport_handle_login(vha, fcport);
+ }
+} /* gnl_event */
+
+static void
+qla24xx_async_gnl_sp_done(void *s, int res)
+{
+ struct srb *sp = s;
+ struct scsi_qla_host *vha = sp->vha;
+ unsigned long flags;
+ struct fc_port *fcport = NULL, *tf;
+ u16 i, n = 0, loop_id;
+ struct event_arg ea;
+ struct get_name_list_extended *e;
+ u64 wwn;
+ struct list_head h;
+
+ ql_dbg(ql_dbg_disc, vha, 0xffff,
+ "Async done-%s res %x mb[1]=%x mb[2]=%x \n",
+ sp->name, res, sp->u.iocb_cmd.u.mbx.in_mb[1],
+ sp->u.iocb_cmd.u.mbx.in_mb[2]);
+
+ memset(&ea, 0, sizeof(ea));
+ ea.sp = sp;
+ ea.rc = res;
+ ea.event = FCME_GNL_DONE;
+
+ if (sp->u.iocb_cmd.u.mbx.in_mb[1] >=
+ sizeof(struct get_name_list_extended)) {
+ n = sp->u.iocb_cmd.u.mbx.in_mb[1] /
+ sizeof(struct get_name_list_extended);
+ ea.data[0] = sp->u.iocb_cmd.u.mbx.in_mb[1]; /* amnt xfered */
+ }
+
+ for (i = 0; i < n; i++) {
+ e = &vha->gnl.l[i];
+ loop_id = le16_to_cpu(e->nport_handle);
+ /* mask out reserve bit */
+ loop_id = (loop_id & 0x7fff);
+ set_bit(loop_id, vha->hw->loop_id_map);
+ wwn = wwn_to_u64(e->port_name);
+
+ ql_dbg(ql_dbg_disc + ql_dbg_verbose, vha, 0xffff,
+ "%s %8phC %02x:%02x:%02x state %d/%d lid %x \n",
+ __func__, (void *)&wwn, e->port_id[2], e->port_id[1],
+ e->port_id[0], e->current_login_state, e->last_login_state,
+ (loop_id & 0x7fff));
+ }
+
+ spin_lock_irqsave(&vha->hw->tgt.sess_lock, flags);
+ vha->gnl.sent = 0;
+
+ INIT_LIST_HEAD(&h);
+ fcport = tf = NULL;
+ if (!list_empty(&vha->gnl.fcports))
+ list_splice_init(&vha->gnl.fcports, &h);
+
+ list_for_each_entry_safe(fcport, tf, &h, gnl_entry) {
+ list_del_init(&fcport->gnl_entry);
+ fcport->flags &= ~FCF_ASYNC_SENT;
+ ea.fcport = fcport;
+
+ qla2x00_fcport_event_handler(vha, &ea);
+ }
+
+ spin_unlock_irqrestore(&vha->hw->tgt.sess_lock, flags);
+
+ sp->free(sp);
+}
+
+int qla24xx_async_gnl(struct scsi_qla_host *vha, fc_port_t *fcport)
+{
+ srb_t *sp;
+ struct srb_iocb *mbx;
+ int rval = QLA_FUNCTION_FAILED;
+ unsigned long flags;
+ u16 *mb;
+
+ if (!vha->flags.online)
+ goto done;
+
+ ql_dbg(ql_dbg_disc, vha, 0xffff,
+ "Async-gnlist WWPN %8phC \n", fcport->port_name);
+
+ spin_lock_irqsave(&vha->hw->tgt.sess_lock, flags);
+ fcport->flags |= FCF_ASYNC_SENT;
+ fcport->disc_state = DSC_GNL;
+ fcport->last_rscn_gen = fcport->rscn_gen;
+ fcport->last_login_gen = fcport->login_gen;
+
+ list_add_tail(&fcport->gnl_entry, &vha->gnl.fcports);
+ if (vha->gnl.sent) {
+ spin_unlock_irqrestore(&vha->hw->tgt.sess_lock, flags);
+ rval = QLA_SUCCESS;
+ goto done;
+ }
+ vha->gnl.sent = 1;
+ spin_unlock_irqrestore(&vha->hw->tgt.sess_lock, flags);
+
+ sp = qla2x00_get_sp(vha, fcport, GFP_KERNEL);
+ if (!sp)
+ goto done;
+ sp->type = SRB_MB_IOCB;
+ sp->name = "gnlist";
+ sp->gen1 = fcport->rscn_gen;
+ sp->gen2 = fcport->login_gen;
+
+ qla2x00_init_timer(sp, qla2x00_get_async_timeout(vha)+2);
+
+ mb = sp->u.iocb_cmd.u.mbx.out_mb;
+ mb[0] = MBC_PORT_NODE_NAME_LIST;
+ mb[1] = BIT_2 | BIT_3;
+ mb[2] = MSW(vha->gnl.ldma);
+ mb[3] = LSW(vha->gnl.ldma);
+ mb[6] = MSW(MSD(vha->gnl.ldma));
+ mb[7] = LSW(MSD(vha->gnl.ldma));
+ mb[8] = vha->gnl.size;
+ mb[9] = vha->vp_idx;
+
+ mbx = &sp->u.iocb_cmd;
+ mbx->timeout = qla2x00_async_iocb_timeout;
+
+ sp->done = qla24xx_async_gnl_sp_done;
+
+ rval = qla2x00_start_sp(sp);
+ if (rval != QLA_SUCCESS)
+ goto done_free_sp;
+
+ ql_dbg(ql_dbg_disc, vha, 0xffff,
+ "Async-%s - OUT WWPN %8phC hndl %x\n",
+ sp->name, fcport->port_name, sp->handle);
+
+ return rval;
+
+done_free_sp:
+ sp->free(sp);
+done:
+ fcport->flags &= ~FCF_ASYNC_SENT;
+ return rval;
+}
+
+int qla24xx_post_gnl_work(struct scsi_qla_host *vha, fc_port_t *fcport)
+{
+ struct qla_work_evt *e;
+
+ e = qla2x00_alloc_work(vha, QLA_EVT_GNL);
+ if (!e)
+ return QLA_FUNCTION_FAILED;
+
+ e->u.fcport.fcport = fcport;
+ return qla2x00_post_work(vha, e);
+}
+
+static
+void qla24xx_async_gpdb_sp_done(void *s, int res)
+{
+ struct srb *sp = s;
+ struct scsi_qla_host *vha = sp->vha;
+ struct qla_hw_data *ha = vha->hw;
+ uint64_t zero = 0;
+ struct port_database_24xx *pd;
+ fc_port_t *fcport = sp->fcport;
+ u16 *mb = sp->u.iocb_cmd.u.mbx.in_mb;
+ int rval = QLA_SUCCESS;
+ struct event_arg ea;
+
+ ql_dbg(ql_dbg_disc, vha, 0xffff,
+ "Async done-%s res %x, WWPN %8phC mb[1]=%x mb[2]=%x \n",
+ sp->name, res, fcport->port_name, mb[1], mb[2]);
+
+ fcport->flags &= ~FCF_ASYNC_SENT;
+
+ if (res) {
+ rval = res;
+ goto gpd_error_out;
+ }
+
+ pd = (struct port_database_24xx *)sp->u.iocb_cmd.u.mbx.in;
+
+ /* Check for logged in state. */
+ if (pd->current_login_state != PDS_PRLI_COMPLETE &&
+ pd->last_login_state != PDS_PRLI_COMPLETE) {
+ ql_dbg(ql_dbg_mbx, vha, 0xffff,
+ "Unable to verify login-state (%x/%x) for "
+ "loop_id %x.\n", pd->current_login_state,
+ pd->last_login_state, fcport->loop_id);
+ rval = QLA_FUNCTION_FAILED;
+ goto gpd_error_out;
+ }
+
+ if (fcport->loop_id == FC_NO_LOOP_ID ||
+ (memcmp(fcport->port_name, (uint8_t *)&zero, 8) &&
+ memcmp(fcport->port_name, pd->port_name, 8))) {
+ /* We lost the device mid way. */
+ rval = QLA_NOT_LOGGED_IN;
+ goto gpd_error_out;
+ }
+
+ /* Names are little-endian. */
+ memcpy(fcport->node_name, pd->node_name, WWN_SIZE);
+
+ /* Get port_id of device. */
+ fcport->d_id.b.domain = pd->port_id[0];
+ fcport->d_id.b.area = pd->port_id[1];
+ fcport->d_id.b.al_pa = pd->port_id[2];
+ fcport->d_id.b.rsvd_1 = 0;
+
+ /* If not target must be initiator or unknown type. */
+ if ((pd->prli_svc_param_word_3[0] & BIT_4) == 0)
+ fcport->port_type = FCT_INITIATOR;
+ else
+ fcport->port_type = FCT_TARGET;
+
+ /* Passback COS information. */
+ fcport->supported_classes = (pd->flags & PDF_CLASS_2) ?
+ FC_COS_CLASS2 : FC_COS_CLASS3;
+
+ if (pd->prli_svc_param_word_3[0] & BIT_7) {
+ fcport->flags |= FCF_CONF_COMP_SUPPORTED;
+ fcport->conf_compl_supported = 1;
+ }
+
+gpd_error_out:
+ memset(&ea, 0, sizeof(ea));
+ ea.event = FCME_GPDB_DONE;
+ ea.rc = rval;
+ ea.fcport = fcport;
+ ea.sp = sp;
+
+ qla2x00_fcport_event_handler(vha, &ea);
+
+ dma_pool_free(ha->s_dma_pool, sp->u.iocb_cmd.u.mbx.in,
+ sp->u.iocb_cmd.u.mbx.in_dma);
+
+ sp->free(sp);
+}
+
+static int qla24xx_post_gpdb_work(struct scsi_qla_host *vha, fc_port_t *fcport,
+ u8 opt)
+{
+ struct qla_work_evt *e;
+
+ e = qla2x00_alloc_work(vha, QLA_EVT_GPDB);
+ if (!e)
+ return QLA_FUNCTION_FAILED;
+
+ e->u.fcport.fcport = fcport;
+ e->u.fcport.opt = opt;
+ return qla2x00_post_work(vha, e);
+}
+
+int qla24xx_async_gpdb(struct scsi_qla_host *vha, fc_port_t *fcport, u8 opt)
+{
+ srb_t *sp;
+ struct srb_iocb *mbx;
+ int rval = QLA_FUNCTION_FAILED;
+ u16 *mb;
+ dma_addr_t pd_dma;
+ struct port_database_24xx *pd;
+ struct qla_hw_data *ha = vha->hw;
+
+ if (!vha->flags.online)
+ goto done;
+
+ fcport->flags |= FCF_ASYNC_SENT;
+ fcport->disc_state = DSC_GPDB;
+
+ sp = qla2x00_get_sp(vha, fcport, GFP_KERNEL);
+ if (!sp)
+ goto done;
+
+ pd = dma_pool_alloc(ha->s_dma_pool, GFP_KERNEL, &pd_dma);
+ if (pd == NULL) {
+ ql_log(ql_log_warn, vha, 0xffff,
+ "Failed to allocate port database structure.\n");
+ goto done_free_sp;
+ }
+ memset(pd, 0, max(PORT_DATABASE_SIZE, PORT_DATABASE_24XX_SIZE));
+
+ sp->type = SRB_MB_IOCB;
+ sp->name = "gpdb";
+ sp->gen1 = fcport->rscn_gen;
+ sp->gen2 = fcport->login_gen;
+ qla2x00_init_timer(sp, qla2x00_get_async_timeout(vha) + 2);
+
+ mb = sp->u.iocb_cmd.u.mbx.out_mb;
+ mb[0] = MBC_GET_PORT_DATABASE;
+ mb[1] = fcport->loop_id;
+ mb[2] = MSW(pd_dma);
+ mb[3] = LSW(pd_dma);
+ mb[6] = MSW(MSD(pd_dma));
+ mb[7] = LSW(MSD(pd_dma));
+ mb[9] = vha->vp_idx;
+ mb[10] = opt;
+
+ mbx = &sp->u.iocb_cmd;
+ mbx->timeout = qla2x00_async_iocb_timeout;
+ mbx->u.mbx.in = (void *)pd;
+ mbx->u.mbx.in_dma = pd_dma;
+
+ sp->done = qla24xx_async_gpdb_sp_done;
+
+ rval = qla2x00_start_sp(sp);
+ if (rval != QLA_SUCCESS)
+ goto done_free_sp;
+
+ ql_dbg(ql_dbg_disc, vha, 0xffff,
+ "Async-%s %8phC hndl %x opt %x\n",
+ sp->name, fcport->port_name, sp->handle, opt);
+
+ return rval;
+
+done_free_sp:
+ if (pd)
+ dma_pool_free(ha->s_dma_pool, pd, pd_dma);
+
+ sp->free(sp);
+done:
+ fcport->flags &= ~FCF_ASYNC_SENT;
+ qla24xx_post_gpdb_work(vha, fcport, opt);
+ return rval;
+}
+
+static
+void qla24xx_handle_gpdb_event(scsi_qla_host_t *vha, struct event_arg *ea)
+{
+ int rval = ea->rc;
+ fc_port_t *fcport = ea->fcport;
+ unsigned long flags;
+
+ fcport->flags &= ~FCF_ASYNC_SENT;
+
+ ql_dbg(ql_dbg_disc, vha, 0xffff,
+ "%s %8phC DS %d LS %d rval %d\n", __func__, fcport->port_name,
+ fcport->disc_state, fcport->fw_login_state, rval);
+
+ if (ea->sp->gen2 != fcport->login_gen) {
+ /* target side must have changed it. */
+ ql_dbg(ql_dbg_disc, vha, 0xffff,
+ "%s %8phC generation changed rscn %d|%d login %d|%d \n",
+ __func__, fcport->port_name, fcport->last_rscn_gen,
+ fcport->rscn_gen, fcport->last_login_gen,
+ fcport->login_gen);
+ return;
+ } else if (ea->sp->gen1 != fcport->rscn_gen) {
+ ql_dbg(ql_dbg_disc, vha, 0xffff, "%s %d %8phC post gidpn\n",
+ __func__, __LINE__, fcport->port_name);
+ qla24xx_post_gidpn_work(vha, fcport);
+ return;
+ }
+
+ if (rval != QLA_SUCCESS) {
+ ql_dbg(ql_dbg_disc, vha, 0xffff, "%s %d %8phC post del sess\n",
+ __func__, __LINE__, fcport->port_name);
+ qlt_schedule_sess_for_deletion_lock(fcport);
+ return;
+ }
+
+ spin_lock_irqsave(&vha->hw->tgt.sess_lock, flags);
+ ea->fcport->login_gen++;
+ ea->fcport->deleted = 0;
+ ea->fcport->logout_on_delete = 1;
+
+ if (!ea->fcport->login_succ && !IS_SW_RESV_ADDR(ea->fcport->d_id)) {
+ vha->fcport_count++;
+ ea->fcport->login_succ = 1;
+
+ if (!IS_IIDMA_CAPABLE(vha->hw) ||
+ !vha->hw->flags.gpsc_supported) {
+ ql_dbg(ql_dbg_disc, vha, 0xffff,
+ "%s %d %8phC post upd_fcport fcp_cnt %d\n",
+ __func__, __LINE__, fcport->port_name,
+ vha->fcport_count);
+
+ qla24xx_post_upd_fcport_work(vha, fcport);
+ } else {
+ ql_dbg(ql_dbg_disc, vha, 0xffff,
+ "%s %d %8phC post gpsc fcp_cnt %d\n",
+ __func__, __LINE__, fcport->port_name,
+ vha->fcport_count);
+
+ qla24xx_post_gpsc_work(vha, fcport);
+ }
+ }
+ spin_unlock_irqrestore(&vha->hw->tgt.sess_lock, flags);
+} /* gpdb event */
+
+int qla24xx_fcport_handle_login(struct scsi_qla_host *vha, fc_port_t *fcport)
+{
+ if (fcport->login_retry == 0)
+ return 0;
+
+ if (fcport->scan_state != QLA_FCPORT_FOUND)
+ return 0;
+
+ ql_dbg(ql_dbg_disc, vha, 0xffff,
+ "%s %8phC DS %d LS %d P %d fl %x confl %p rscn %d|%d login %d|%d retry %d lid %d\n",
+ __func__, fcport->port_name, fcport->disc_state,
+ fcport->fw_login_state, fcport->login_pause, fcport->flags,
+ fcport->conflict, fcport->last_rscn_gen, fcport->rscn_gen,
+ fcport->last_login_gen, fcport->login_gen, fcport->login_retry,
+ fcport->loop_id);
+
+ fcport->login_retry--;
+
+ if ((fcport->fw_login_state == DSC_LS_PLOGI_PEND) ||
+ (fcport->fw_login_state == DSC_LS_PLOGI_COMP) ||
+ (fcport->fw_login_state == DSC_LS_PRLI_PEND))
+ return 0;
+
+ /* for pure Target Mode. Login will not be initiated */
+ if (vha->host->active_mode == MODE_TARGET)
+ return 0;
+
+ if (fcport->flags & FCF_ASYNC_SENT) {
+ set_bit(RELOGIN_NEEDED, &vha->dpc_flags);
+ return 0;
+ }
+
+ switch (fcport->disc_state) {
+ case DSC_DELETED:
+ if (fcport->loop_id == FC_NO_LOOP_ID) {
+ ql_dbg(ql_dbg_disc, vha, 0xffff,
+ "%s %d %8phC post gnl\n",
+ __func__, __LINE__, fcport->port_name);
+ qla24xx_async_gnl(vha, fcport);
+ } else {
+ ql_dbg(ql_dbg_disc, vha, 0xffff,
+ "%s %d %8phC post login\n",
+ __func__, __LINE__, fcport->port_name);
+ fcport->disc_state = DSC_LOGIN_PEND;
+ qla2x00_post_async_login_work(vha, fcport, NULL);
+ }
+ break;
+
+ case DSC_GNL:
+ if (fcport->login_pause) {
+ fcport->last_rscn_gen = fcport->rscn_gen;
+ fcport->last_login_gen = fcport->login_gen;
+ set_bit(RELOGIN_NEEDED, &vha->dpc_flags);
+ break;
+ }
+
+ if (fcport->flags & FCF_FCP2_DEVICE) {
+ u8 opt = PDO_FORCE_ADISC;
+
+ ql_dbg(ql_dbg_disc, vha, 0xffff,
+ "%s %d %8phC post gpdb\n",
+ __func__, __LINE__, fcport->port_name);
+
+ fcport->disc_state = DSC_GPDB;
+ qla24xx_post_gpdb_work(vha, fcport, opt);
+ } else {
+ ql_dbg(ql_dbg_disc, vha, 0xffff,
+ "%s %d %8phC post login \n",
+ __func__, __LINE__, fcport->port_name);
+ fcport->disc_state = DSC_LOGIN_PEND;
+ qla2x00_post_async_login_work(vha, fcport, NULL);
+ }
+
+ break;
+
+ case DSC_LOGIN_FAILED:
+ ql_dbg(ql_dbg_disc, vha, 0xffff,
+ "%s %d %8phC post gidpn \n",
+ __func__, __LINE__, fcport->port_name);
+
+ qla24xx_post_gidpn_work(vha, fcport);
+ break;
+
+ case DSC_LOGIN_COMPLETE:
+ /* recheck login state */
+ ql_dbg(ql_dbg_disc, vha, 0xffff,
+ "%s %d %8phC post gpdb \n",
+ __func__, __LINE__, fcport->port_name);
+
+ qla24xx_post_gpdb_work(vha, fcport, PDO_FORCE_ADISC);
+ break;
+
+ default:
+ break;
+ }
+
+ return 0;
+}
+
+static
+void qla24xx_handle_rscn_event(fc_port_t *fcport, struct event_arg *ea)
+{
+ fcport->rscn_gen++;
+
+ ql_dbg(ql_dbg_disc, fcport->vha, 0xffff,
+ "%s %8phC DS %d LS %d\n",
+ __func__, fcport->port_name, fcport->disc_state,
+ fcport->fw_login_state);
+
+ if (fcport->flags & FCF_ASYNC_SENT)
+ return;
+
+ switch (fcport->disc_state) {
+ case DSC_DELETED:
+ case DSC_LOGIN_COMPLETE:
+ qla24xx_post_gidpn_work(fcport->vha, fcport);
+ break;
+
+ default:
+ break;
+ }
+}
+
+int qla24xx_post_newsess_work(struct scsi_qla_host *vha, port_id_t *id,
+ u8 *port_name, void *pla)
+{
+ struct qla_work_evt *e;
+ e = qla2x00_alloc_work(vha, QLA_EVT_NEW_SESS);
+ if (!e)
+ return QLA_FUNCTION_FAILED;
+
+ e->u.new_sess.id = *id;
+ e->u.new_sess.pla = pla;
+ memcpy(e->u.new_sess.port_name, port_name, WWN_SIZE);
+
+ return qla2x00_post_work(vha, e);
+}
+
+static
+int qla24xx_handle_delete_done_event(scsi_qla_host_t *vha,
+ struct event_arg *ea)
+{
+ fc_port_t *fcport = ea->fcport;
+
+ if (test_bit(UNLOADING, &vha->dpc_flags))
+ return 0;
+
+ switch (vha->host->active_mode) {
+ case MODE_INITIATOR:
+ case MODE_DUAL:
+ if (fcport->scan_state == QLA_FCPORT_FOUND)
+ qla24xx_fcport_handle_login(vha, fcport);
+ break;
+
+ case MODE_TARGET:
+ default:
+ /* no-op */
+ break;
+ }
+
+ return 0;
+}
+
+static
+void qla24xx_handle_relogin_event(scsi_qla_host_t *vha,
+ struct event_arg *ea)
+{
+ fc_port_t *fcport = ea->fcport;
+
+ if (fcport->scan_state != QLA_FCPORT_FOUND) {
+ fcport->login_retry++;
+ return;
+ }
+
+ ql_dbg(ql_dbg_disc, vha, 0xffff,
+ "%s %8phC DS %d LS %d P %d del %d cnfl %p rscn %d|%d login %d|%d fl %x\n",
+ __func__, fcport->port_name, fcport->disc_state,
+ fcport->fw_login_state, fcport->login_pause,
+ fcport->deleted, fcport->conflict,
+ fcport->last_rscn_gen, fcport->rscn_gen,
+ fcport->last_login_gen, fcport->login_gen,
+ fcport->flags);
+
+ if ((fcport->fw_login_state == DSC_LS_PLOGI_PEND) ||
+ (fcport->fw_login_state == DSC_LS_PLOGI_COMP) ||
+ (fcport->fw_login_state == DSC_LS_PRLI_PEND))
+ return;
+
+ if (fcport->flags & FCF_ASYNC_SENT) {
+ fcport->login_retry++;
+ set_bit(RELOGIN_NEEDED, &vha->dpc_flags);
+ return;
+ }
+
+ if (fcport->disc_state == DSC_DELETE_PEND) {
+ fcport->login_retry++;
+ return;
+ }
+
+ if (fcport->last_rscn_gen != fcport->rscn_gen) {
+ ql_dbg(ql_dbg_disc, vha, 0xffff, "%s %d %8phC post gidpn\n",
+ __func__, __LINE__, fcport->port_name);
+
+ qla24xx_async_gidpn(vha, fcport);
+ return;
+ }
+
+ qla24xx_fcport_handle_login(vha, fcport);
+}
+
+void qla2x00_fcport_event_handler(scsi_qla_host_t *vha, struct event_arg *ea)
+{
+ fc_port_t *fcport, *f, *tf;
+ uint32_t id = 0, mask, rid;
+ int rc;
+
+ switch (ea->event) {
+ case FCME_RELOGIN:
+ if (test_bit(UNLOADING, &vha->dpc_flags))
+ return;
-done_free_sp:
- sp->free(fcport->vha, sp);
-done:
- return rval;
+ qla24xx_handle_relogin_event(vha, ea);
+ break;
+ case FCME_RSCN:
+ if (test_bit(UNLOADING, &vha->dpc_flags))
+ return;
+ switch (ea->id.b.rsvd_1) {
+ case RSCN_PORT_ADDR:
+ fcport = qla2x00_find_fcport_by_nportid(vha, &ea->id, 1);
+ if (!fcport) {
+ /* cable moved */
+ rc = qla24xx_post_gpnid_work(vha, &ea->id);
+ if (rc) {
+ ql_log(ql_log_warn, vha, 0xffff,
+ "RSCN GPNID work failed %02x%02x%02x\n",
+ ea->id.b.domain, ea->id.b.area,
+ ea->id.b.al_pa);
+ }
+ } else {
+ ea->fcport = fcport;
+ qla24xx_handle_rscn_event(fcport, ea);
+ }
+ break;
+ case RSCN_AREA_ADDR:
+ case RSCN_DOM_ADDR:
+ if (ea->id.b.rsvd_1 == RSCN_AREA_ADDR) {
+ mask = 0xffff00;
+ ql_log(ql_dbg_async, vha, 0xffff,
+ "RSCN: Area 0x%06x was affected\n",
+ ea->id.b24);
+ } else {
+ mask = 0xff0000;
+ ql_log(ql_dbg_async, vha, 0xffff,
+ "RSCN: Domain 0x%06x was affected\n",
+ ea->id.b24);
+ }
+
+ rid = ea->id.b24 & mask;
+ list_for_each_entry_safe(f, tf, &vha->vp_fcports,
+ list) {
+ id = f->d_id.b24 & mask;
+ if (rid == id) {
+ ea->fcport = f;
+ qla24xx_handle_rscn_event(f, ea);
+ }
+ }
+ break;
+ case RSCN_FAB_ADDR:
+ default:
+ ql_log(ql_log_warn, vha, 0xffff,
+ "RSCN: Fabric was affected. Addr format %d\n",
+ ea->id.b.rsvd_1);
+ qla2x00_mark_all_devices_lost(vha, 1);
+ set_bit(LOOP_RESYNC_NEEDED, &vha->dpc_flags);
+ set_bit(LOCAL_LOOP_UPDATE, &vha->dpc_flags);
+ }
+ break;
+ case FCME_GIDPN_DONE:
+ qla24xx_handle_gidpn_event(vha, ea);
+ break;
+ case FCME_GNL_DONE:
+ qla24xx_handle_gnl_done_event(vha, ea);
+ break;
+ case FCME_GPSC_DONE:
+ qla24xx_post_upd_fcport_work(vha, ea->fcport);
+ break;
+ case FCME_PLOGI_DONE: /* Initiator side sent LLIOCB */
+ qla24xx_handle_plogi_done_event(vha, ea);
+ break;
+ case FCME_GPDB_DONE:
+ qla24xx_handle_gpdb_event(vha, ea);
+ break;
+ case FCME_GPNID_DONE:
+ qla24xx_handle_gpnid_event(vha, ea);
+ break;
+ case FCME_DELETE_DONE:
+ qla24xx_handle_delete_done_event(vha, ea);
+ break;
+ default:
+ BUG_ON(1);
+ break;
+ }
}
static void
qla2x00_tmf_iocb_timeout(void *data)
{
- srb_t *sp = (srb_t *)data;
+ srb_t *sp = data;
struct srb_iocb *tmf = &sp->u.iocb_cmd;
tmf->u.tmf.comp_status = CS_TIMEOUT;
}
static void
-qla2x00_tmf_sp_done(void *data, void *ptr, int res)
+qla2x00_tmf_sp_done(void *ptr, int res)
{
- srb_t *sp = (srb_t *)ptr;
+ srb_t *sp = ptr;
struct srb_iocb *tmf = &sp->u.iocb_cmd;
+
complete(&tmf->u.tmf.comp);
}
}
done_free_sp:
- sp->free(vha, sp);
+ sp->free(sp);
done:
return rval;
}
static void
qla24xx_abort_iocb_timeout(void *data)
{
- srb_t *sp = (srb_t *)data;
+ srb_t *sp = data;
struct srb_iocb *abt = &sp->u.iocb_cmd;
abt->u.abt.comp_status = CS_TIMEOUT;
}
static void
-qla24xx_abort_sp_done(void *data, void *ptr, int res)
+qla24xx_abort_sp_done(void *ptr, int res)
{
- srb_t *sp = (srb_t *)ptr;
+ srb_t *sp = ptr;
struct srb_iocb *abt = &sp->u.iocb_cmd;
complete(&abt->u.abt.comp);
static int
qla24xx_async_abort_cmd(srb_t *cmd_sp)
{
- scsi_qla_host_t *vha = cmd_sp->fcport->vha;
+ scsi_qla_host_t *vha = cmd_sp->vha;
fc_port_t *fcport = cmd_sp->fcport;
struct srb_iocb *abt_iocb;
srb_t *sp;
QLA_SUCCESS : QLA_FUNCTION_FAILED;
done_free_sp:
- sp->free(vha, sp);
+ sp->free(sp);
done:
return rval;
}
return qla24xx_async_abort_cmd(sp);
}
-void
-qla2x00_async_login_done(struct scsi_qla_host *vha, fc_port_t *fcport,
- uint16_t *data)
+static void
+qla24xx_handle_plogi_done_event(struct scsi_qla_host *vha, struct event_arg *ea)
{
- int rval;
+ port_id_t cid; /* conflict Nport id */
- switch (data[0]) {
+ switch (ea->data[0]) {
case MBS_COMMAND_COMPLETE:
/*
* Driver must validate login state - If PRLI not complete,
* force a relogin attempt via implicit LOGO, PLOGI, and PRLI
* requests.
*/
- rval = qla2x00_get_port_database(vha, fcport, 0);
- if (rval == QLA_NOT_LOGGED_IN) {
- fcport->flags &= ~FCF_ASYNC_SENT;
- fcport->flags |= FCF_LOGIN_NEEDED;
- set_bit(RELOGIN_NEEDED, &vha->dpc_flags);
- break;
- }
-
- if (rval != QLA_SUCCESS) {
- qla2x00_post_async_logout_work(vha, fcport, NULL);
- qla2x00_post_async_login_work(vha, fcport, NULL);
- break;
- }
- if (fcport->flags & FCF_FCP2_DEVICE) {
- qla2x00_post_async_adisc_work(vha, fcport, data);
- break;
- }
- qla2x00_update_fcport(vha, fcport);
+ ql_dbg(ql_dbg_disc, vha, 0xffff,
+ "%s %d %8phC post gpdb\n",
+ __func__, __LINE__, ea->fcport->port_name);
+ ea->fcport->chip_reset = vha->hw->chip_reset;
+ ea->fcport->logout_on_delete = 1;
+ qla24xx_post_gpdb_work(vha, ea->fcport, 0);
break;
case MBS_COMMAND_ERROR:
- fcport->flags &= ~FCF_ASYNC_SENT;
- if (data[1] & QLA_LOGIO_LOGIN_RETRIED)
+ ql_dbg(ql_dbg_disc, vha, 0xffff, "%s %d %8phC cmd error %x\n",
+ __func__, __LINE__, ea->fcport->port_name, ea->data[1]);
+
+ ea->fcport->flags &= ~FCF_ASYNC_SENT;
+ ea->fcport->disc_state = DSC_LOGIN_FAILED;
+ if (ea->data[1] & QLA_LOGIO_LOGIN_RETRIED)
set_bit(RELOGIN_NEEDED, &vha->dpc_flags);
else
- qla2x00_mark_device_lost(vha, fcport, 1, 0);
- break;
- case MBS_PORT_ID_USED:
- fcport->loop_id = data[1];
- qla2x00_post_async_logout_work(vha, fcport, NULL);
- qla2x00_post_async_login_work(vha, fcport, NULL);
+ qla2x00_mark_device_lost(vha, ea->fcport, 1, 0);
break;
case MBS_LOOP_ID_USED:
- fcport->loop_id++;
- rval = qla2x00_find_new_loop_id(vha, fcport);
- if (rval != QLA_SUCCESS) {
- fcport->flags &= ~FCF_ASYNC_SENT;
- qla2x00_mark_device_lost(vha, fcport, 1, 0);
- break;
+ /* data[1] = IO PARAM 1 = nport ID */
+ cid.b.domain = (ea->iop[1] >> 16) & 0xff;
+ cid.b.area = (ea->iop[1] >> 8) & 0xff;
+ cid.b.al_pa = ea->iop[1] & 0xff;
+ cid.b.rsvd_1 = 0;
+
+ ql_dbg(ql_dbg_disc, vha, 0xffff,
+ "%s %d %8phC LoopID 0x%x in use post gnl\n",
+ __func__, __LINE__, ea->fcport->port_name,
+ ea->fcport->loop_id);
+
+ if (IS_SW_RESV_ADDR(cid)) {
+ set_bit(ea->fcport->loop_id, vha->hw->loop_id_map);
+ ea->fcport->loop_id = FC_NO_LOOP_ID;
+ } else {
+ qla2x00_clear_loop_id(ea->fcport);
}
- qla2x00_post_async_login_work(vha, fcport, NULL);
+ qla24xx_post_gnl_work(vha, ea->fcport);
+ break;
+ case MBS_PORT_ID_USED:
+ ql_dbg(ql_dbg_disc, vha, 0xffff,
+ "%s %d %8phC NPortId %02x%02x%02x inuse post gidpn\n",
+ __func__, __LINE__, ea->fcport->port_name,
+ ea->fcport->d_id.b.domain, ea->fcport->d_id.b.area,
+ ea->fcport->d_id.b.al_pa);
+
+ qla2x00_clear_loop_id(ea->fcport);
+ qla24xx_post_gidpn_work(vha, ea->fcport);
break;
}
return;
qla2x00_async_logout_done(struct scsi_qla_host *vha, fc_port_t *fcport,
uint16_t *data)
{
- /* Don't re-login in target mode */
- if (!fcport->tgt_session)
- qla2x00_mark_device_lost(vha, fcport, 1, 0);
+ qla2x00_mark_device_lost(vha, fcport, 1, 0);
qlt_logo_completion_handler(fcport, data[0]);
+ fcport->login_gen++;
return;
}
}
}
- if (qla_ini_mode_enabled(vha))
+ if (qla_ini_mode_enabled(vha) || qla_dual_mode_enabled(vha))
rval = qla2x00_init_rings(vha);
ha->flags.chip_reset_done = 1;
__func__, ha->fw_options[2]);
}
+ /* Move PUREX, ABTS RX & RIDA to ATIOQ */
+ if (ql2xmvasynctoatio) {
+ if (qla_tgt_mode_enabled(vha) ||
+ qla_dual_mode_enabled(vha))
+ ha->fw_options[2] |= BIT_11;
+ else
+ ha->fw_options[2] &= ~BIT_11;
+ }
+
+ ql_dbg(ql_dbg_init, vha, 0xffff,
+ "%s, add FW options 1-3 = 0x%04x 0x%04x 0x%04x mode %x\n",
+ __func__, ha->fw_options[1], ha->fw_options[2],
+ ha->fw_options[3], vha->host->active_mode);
+ qla2x00_set_fw_options(vha, ha->fw_options);
+
/* Update Serial Link options. */
if ((le16_to_cpu(ha->fw_seriallink_options24[0]) & BIT_0) == 0)
return;
rport = fcport->drport ? fcport->drport: fcport->rport;
fcport->drport = NULL;
spin_unlock_irqrestore(fcport->vha->host->host_lock, flags);
- if (rport)
+ if (rport) {
+ ql_dbg(ql_dbg_disc, fcport->vha, 0xffff,
+ "%s %8phN. rport %p roles %x \n",
+ __func__, fcport->port_name, rport,
+ rport->roles);
+
fc_remote_port_delete(rport);
+ }
}
/**
qla2x00_set_fcport_state(fcport, FCS_UNCONFIGURED);
fcport->supported_classes = FC_COS_UNSPECIFIED;
+ fcport->ct_desc.ct_sns = dma_alloc_coherent(&vha->hw->pdev->dev,
+ sizeof(struct ct_sns_pkt), &fcport->ct_desc.ct_sns_dma,
+ flags);
+ fcport->disc_state = DSC_DELETED;
+ fcport->fw_login_state = DSC_LS_PORT_UNAVAIL;
+ fcport->deleted = QLA_SESS_DELETED;
+ fcport->login_retry = vha->hw->login_retry_count;
+ fcport->login_retry = 5;
+ fcport->logout_on_delete = 1;
+
+ if (!fcport->ct_desc.ct_sns) {
+ ql_log(ql_log_warn, vha, 0xffff,
+ "Failed to allocate ct_sns request.\n");
+ kfree(fcport);
+ fcport = NULL;
+ }
+ INIT_WORK(&fcport->del_work, qla24xx_delete_sess_fn);
+ INIT_LIST_HEAD(&fcport->gnl_entry);
+ INIT_LIST_HEAD(&fcport->list);
+
return fcport;
}
+void
+qla2x00_free_fcport(fc_port_t *fcport)
+{
+ if (fcport->ct_desc.ct_sns) {
+ dma_free_coherent(&fcport->vha->hw->pdev->dev,
+ sizeof(struct ct_sns_pkt), fcport->ct_desc.ct_sns,
+ fcport->ct_desc.ct_sns_dma);
+
+ fcport->ct_desc.ct_sns = NULL;
+ }
+ kfree(fcport);
+}
+
/*
* qla2x00_configure_loop
* Updates Fibre Channel Device Database with what is actually on loop.
} else if (ha->current_topology == ISP_CFG_N) {
clear_bit(RSCN_UPDATE, &flags);
-
+ } else if (ha->current_topology == ISP_CFG_NL) {
+ clear_bit(RSCN_UPDATE, &flags);
+ set_bit(LOCAL_LOOP_UPDATE, &flags);
} else if (!vha->flags.online ||
(test_bit(ABORT_ISP_ACTIVE, &flags))) {
-
set_bit(RSCN_UPDATE, &flags);
set_bit(LOCAL_LOOP_UPDATE, &flags);
}
* Process any ATIO queue entries that came in
* while we weren't online.
*/
- if (qla_tgt_mode_enabled(vha)) {
+ if (qla_tgt_mode_enabled(vha) ||
+ qla_dual_mode_enabled(vha)) {
if (IS_QLA27XX(ha) || IS_QLA83XX(ha)) {
spin_lock_irqsave(&ha->tgt.atio_lock,
flags);
uint16_t loop_id;
uint8_t domain, area, al_pa;
struct qla_hw_data *ha = vha->hw;
+ unsigned long flags;
found_devs = 0;
new_fcport = NULL;
"Marking port lost loop_id=0x%04x.\n",
fcport->loop_id);
- qla2x00_set_fcport_state(fcport, FCS_DEVICE_LOST);
+ qla2x00_mark_device_lost(vha, fcport, 0, 0);
}
}
if (loop_id > LAST_LOCAL_LOOP_ID)
continue;
- memset(new_fcport, 0, sizeof(fc_port_t));
+ memset(new_fcport->port_name, 0, WWN_SIZE);
/* Fill in member data. */
new_fcport->d_id.b.domain = domain;
new_fcport->d_id.b.area = area;
new_fcport->d_id.b.al_pa = al_pa;
new_fcport->loop_id = loop_id;
+
rval2 = qla2x00_get_port_database(vha, new_fcport, 0);
if (rval2 != QLA_SUCCESS) {
ql_dbg(ql_dbg_disc, vha, 0x201a,
continue;
}
+ spin_lock_irqsave(&vha->hw->tgt.sess_lock, flags);
/* Check for matching device in port list. */
found = 0;
fcport = NULL;
memcpy(fcport->node_name, new_fcport->node_name,
WWN_SIZE);
+ if (!fcport->login_succ) {
+ vha->fcport_count++;
+ fcport->login_succ = 1;
+ fcport->disc_state = DSC_LOGIN_COMPLETE;
+ }
+
found++;
break;
}
/* Allocate a new replacement fcport. */
fcport = new_fcport;
+ if (!fcport->login_succ) {
+ vha->fcport_count++;
+ fcport->login_succ = 1;
+ fcport->disc_state = DSC_LOGIN_COMPLETE;
+ }
+
+ spin_unlock_irqrestore(&vha->hw->tgt.sess_lock, flags);
+
new_fcport = qla2x00_alloc_fcport(vha, GFP_KERNEL);
+
if (new_fcport == NULL) {
ql_log(ql_log_warn, vha, 0x201c,
"Failed to allocate memory for fcport.\n");
rval = QLA_MEMORY_ALLOC_FAILED;
goto cleanup_allocation;
}
+ spin_lock_irqsave(&vha->hw->tgt.sess_lock, flags);
new_fcport->flags &= ~FCF_FABRIC_DEVICE;
}
+ spin_unlock_irqrestore(&vha->hw->tgt.sess_lock, flags);
+
/* Base iIDMA settings on HBA port speed. */
fcport->fp_speed = ha->link_data_rate;
}
}
+/* qla2x00_reg_remote_port is reserved for Initiator Mode only.*/
static void
qla2x00_reg_remote_port(scsi_qla_host_t *vha, fc_port_t *fcport)
{
"Unable to allocate fc remote port.\n");
return;
}
- /*
- * Create target mode FC NEXUS in qla_target.c if target mode is
- * enabled..
- */
-
- qlt_fc_port_added(vha, fcport);
spin_lock_irqsave(fcport->vha->host->host_lock, flags);
*((fc_port_t **)rport->dd_data) = fcport;
rport_ids.roles |= FC_RPORT_ROLE_FCP_INITIATOR;
if (fcport->port_type == FCT_TARGET)
rport_ids.roles |= FC_RPORT_ROLE_FCP_TARGET;
+
+ ql_dbg(ql_dbg_disc, vha, 0xffff,
+ "%s %8phN. rport %p is %s mode \n",
+ __func__, fcport->port_name, rport,
+ (fcport->port_type == FCT_TARGET) ? "tgt" : "ini");
+
fc_remote_port_rolechg(rport, rport_ids.roles);
}
{
fcport->vha = vha;
+ if (IS_SW_RESV_ADDR(fcport->d_id))
+ return;
+
+ ql_dbg(ql_dbg_disc, vha, 0xffff, "%s %8phC \n",
+ __func__, fcport->port_name);
+
if (IS_QLAFX00(vha->hw)) {
qla2x00_set_fcport_state(fcport, FCS_ONLINE);
goto reg_port;
}
fcport->login_retry = 0;
fcport->flags &= ~(FCF_LOGIN_NEEDED | FCF_ASYNC_SENT);
+ fcport->disc_state = DSC_LOGIN_COMPLETE;
+ fcport->deleted = 0;
+ fcport->logout_on_delete = 1;
qla2x00_set_fcport_state(fcport, FCS_ONLINE);
qla2x00_iidma_fcport(vha, fcport);
qla24xx_update_fcport_fcp_prio(vha, fcport);
reg_port:
- if (qla_ini_mode_enabled(vha))
+ switch (vha->host->active_mode) {
+ case MODE_INITIATOR:
qla2x00_reg_remote_port(vha, fcport);
- else {
- /*
- * Create target mode FC NEXUS in qla_target.c
- */
- qlt_fc_port_added(vha, fcport);
+ break;
+ case MODE_TARGET:
+ if (!vha->vha_tgt.qla_tgt->tgt_stop &&
+ !vha->vha_tgt.qla_tgt->tgt_stopped)
+ qlt_fc_port_added(vha, fcport);
+ break;
+ case MODE_DUAL:
+ qla2x00_reg_remote_port(vha, fcport);
+ if (!vha->vha_tgt.qla_tgt->tgt_stop &&
+ !vha->vha_tgt.qla_tgt->tgt_stopped)
+ qlt_fc_port_added(vha, fcport);
+ break;
+ default:
+ break;
}
}
qla2x00_configure_fabric(scsi_qla_host_t *vha)
{
int rval;
- fc_port_t *fcport, *fcptemp;
- uint16_t next_loopid;
+ fc_port_t *fcport;
uint16_t mb[MAILBOX_REGISTER_COUNT];
uint16_t loop_id;
LIST_HEAD(new_fcports);
struct qla_hw_data *ha = vha->hw;
- struct scsi_qla_host *base_vha = pci_get_drvdata(ha->pdev);
int discovery_gen;
/* If FL port exists, then SNS is present */
}
vha->device_flags |= SWITCH_FOUND;
+
+ if (qla_tgt_mode_enabled(vha) || qla_dual_mode_enabled(vha)) {
+ rval = qla2x00_send_change_request(vha, 0x3, 0);
+ if (rval != QLA_SUCCESS)
+ ql_log(ql_log_warn, vha, 0x121,
+ "Failed to enable receiving of RSCN requests: 0x%x.\n",
+ rval);
+ }
+
+
do {
+ qla2x00_mgmt_svr_login(vha);
+
/* FDMI support. */
if (ql2xfdmienable &&
test_and_clear_bit(REGISTER_FDMI_NEEDED, &vha->dpc_flags))
}
}
-#define QLA_FCPORT_SCAN 1
-#define QLA_FCPORT_FOUND 2
-
list_for_each_entry(fcport, &vha->vp_fcports, list) {
fcport->scan_state = QLA_FCPORT_SCAN;
}
* will be newer than discovery_gen. */
qlt_do_generation_tick(vha, &discovery_gen);
- rval = qla2x00_find_all_fabric_devs(vha, &new_fcports);
+ rval = qla2x00_find_all_fabric_devs(vha);
if (rval != QLA_SUCCESS)
break;
-
- /*
- * Logout all previous fabric devices marked lost, except
- * FCP2 devices.
- */
- list_for_each_entry(fcport, &vha->vp_fcports, list) {
- if (test_bit(LOOP_RESYNC_NEEDED, &vha->dpc_flags))
- break;
-
- if ((fcport->flags & FCF_FABRIC_DEVICE) == 0)
- continue;
-
- if (fcport->scan_state == QLA_FCPORT_SCAN) {
- if (qla_ini_mode_enabled(base_vha) &&
- atomic_read(&fcport->state) == FCS_ONLINE) {
- qla2x00_mark_device_lost(vha, fcport,
- ql2xplogiabsentdevice, 0);
- if (fcport->loop_id != FC_NO_LOOP_ID &&
- (fcport->flags & FCF_FCP2_DEVICE) == 0 &&
- fcport->port_type != FCT_INITIATOR &&
- fcport->port_type != FCT_BROADCAST) {
- ha->isp_ops->fabric_logout(vha,
- fcport->loop_id,
- fcport->d_id.b.domain,
- fcport->d_id.b.area,
- fcport->d_id.b.al_pa);
- qla2x00_clear_loop_id(fcport);
- }
- } else if (!qla_ini_mode_enabled(base_vha)) {
- /*
- * In target mode, explicitly kill
- * sessions and log out of devices
- * that are gone, so that we don't
- * end up with an initiator using the
- * wrong ACL (if the fabric recycles
- * an FC address and we have a stale
- * session around) and so that we don't
- * report initiators that are no longer
- * on the fabric.
- */
- ql_dbg(ql_dbg_tgt_mgt, vha, 0xf077,
- "port gone, logging out/killing session: "
- "%8phC state 0x%x flags 0x%x fc4_type 0x%x "
- "scan_state %d\n",
- fcport->port_name,
- atomic_read(&fcport->state),
- fcport->flags, fcport->fc4_type,
- fcport->scan_state);
- qlt_fc_port_deleted(vha, fcport,
- discovery_gen);
- }
- }
- }
-
- /* Starting free loop ID. */
- next_loopid = ha->min_external_loopid;
-
- /*
- * Scan through our port list and login entries that need to be
- * logged in.
- */
- list_for_each_entry(fcport, &vha->vp_fcports, list) {
- if (atomic_read(&vha->loop_down_timer) ||
- test_bit(LOOP_RESYNC_NEEDED, &vha->dpc_flags))
- break;
-
- if ((fcport->flags & FCF_FABRIC_DEVICE) == 0 ||
- (fcport->flags & FCF_LOGIN_NEEDED) == 0)
- continue;
-
- /*
- * If we're not an initiator, skip looking for devices
- * and logging in. There's no reason for us to do it,
- * and it seems to actively cause problems in target
- * mode if we race with the initiator logging into us
- * (we might get the "port ID used" status back from
- * our login command and log out the initiator, which
- * seems to cause havoc).
- */
- if (!qla_ini_mode_enabled(base_vha)) {
- if (fcport->scan_state == QLA_FCPORT_FOUND) {
- ql_dbg(ql_dbg_tgt_mgt, vha, 0xf078,
- "port %8phC state 0x%x flags 0x%x fc4_type 0x%x "
- "scan_state %d (initiator mode disabled; skipping "
- "login)\n", fcport->port_name,
- atomic_read(&fcport->state),
- fcport->flags, fcport->fc4_type,
- fcport->scan_state);
- }
- continue;
- }
-
- if (fcport->loop_id == FC_NO_LOOP_ID) {
- fcport->loop_id = next_loopid;
- rval = qla2x00_find_new_loop_id(
- base_vha, fcport);
- if (rval != QLA_SUCCESS) {
- /* Ran out of IDs to use */
- break;
- }
- }
- /* Login and update database */
- qla2x00_fabric_dev_login(vha, fcport, &next_loopid);
- }
-
- /* Exit if out of loop IDs. */
- if (rval != QLA_SUCCESS) {
- break;
- }
-
- /*
- * Login and add the new devices to our port list.
- */
- list_for_each_entry_safe(fcport, fcptemp, &new_fcports, list) {
- if (atomic_read(&vha->loop_down_timer) ||
- test_bit(LOOP_RESYNC_NEEDED, &vha->dpc_flags))
- break;
-
- /*
- * If we're not an initiator, skip looking for devices
- * and logging in. There's no reason for us to do it,
- * and it seems to actively cause problems in target
- * mode if we race with the initiator logging into us
- * (we might get the "port ID used" status back from
- * our login command and log out the initiator, which
- * seems to cause havoc).
- */
- if (qla_ini_mode_enabled(base_vha)) {
- /* Find a new loop ID to use. */
- fcport->loop_id = next_loopid;
- rval = qla2x00_find_new_loop_id(base_vha,
- fcport);
- if (rval != QLA_SUCCESS) {
- /* Ran out of IDs to use */
- break;
- }
-
- /* Login and update database */
- qla2x00_fabric_dev_login(vha, fcport,
- &next_loopid);
- } else {
- ql_dbg(ql_dbg_tgt_mgt, vha, 0xf079,
- "new port %8phC state 0x%x flags 0x%x fc4_type "
- "0x%x scan_state %d (initiator mode disabled; "
- "skipping login)\n",
- fcport->port_name,
- atomic_read(&fcport->state),
- fcport->flags, fcport->fc4_type,
- fcport->scan_state);
- }
-
- list_move_tail(&fcport->list, &vha->vp_fcports);
- }
} while (0);
- /* Free all new device structures not processed. */
- list_for_each_entry_safe(fcport, fcptemp, &new_fcports, list) {
- list_del(&fcport->list);
- kfree(fcport);
- }
-
- if (rval) {
+ if (rval)
ql_dbg(ql_dbg_disc, vha, 0x2068,
"Configure fabric error exit rval=%d.\n", rval);
- }
return (rval);
}
* Kernel context.
*/
static int
-qla2x00_find_all_fabric_devs(scsi_qla_host_t *vha,
- struct list_head *new_fcports)
+qla2x00_find_all_fabric_devs(scsi_qla_host_t *vha)
{
int rval;
uint16_t loop_id;
- fc_port_t *fcport, *new_fcport, *fcptemp;
+ fc_port_t *fcport, *new_fcport;
int found;
sw_info_t *swl;
port_id_t wrap = {}, nxt_d_id;
struct qla_hw_data *ha = vha->hw;
struct scsi_qla_host *base_vha = pci_get_drvdata(ha->pdev);
+ unsigned long flags;
rval = QLA_SUCCESS;
swl = NULL;
} else if (qla2x00_gnn_id(vha, swl) != QLA_SUCCESS) {
swl = NULL;
- } else if (ql2xiidmaenable &&
- qla2x00_gfpn_id(vha, swl) == QLA_SUCCESS) {
- qla2x00_gpsc(vha, swl);
+ } else if (qla2x00_gfpn_id(vha, swl) != QLA_SUCCESS) {
+ swl = NULL;
}
/* If other queries succeeded probe for FC-4 type */
ql_log(ql_log_warn, vha, 0x2064,
"SNS scan failed -- assuming "
"zero-entry result.\n");
- list_for_each_entry_safe(fcport, fcptemp,
- new_fcports, list) {
- list_del(&fcport->list);
- kfree(fcport);
- }
rval = QLA_SUCCESS;
break;
}
new_fcport->fc4_type != FC4_TYPE_UNKNOWN))
continue;
+ spin_lock_irqsave(&vha->hw->tgt.sess_lock, flags);
+
/* Locate matching device in database. */
found = 0;
list_for_each_entry(fcport, &vha->vp_fcports, list) {
*/
if (fcport->d_id.b24 == new_fcport->d_id.b24 &&
(atomic_read(&fcport->state) == FCS_ONLINE ||
- !qla_ini_mode_enabled(base_vha))) {
+ (vha->host->active_mode == MODE_TARGET))) {
break;
}
* Log it out if still logged in and mark it for
* relogin later.
*/
- if (!qla_ini_mode_enabled(base_vha)) {
+ if (qla_tgt_mode_enabled(base_vha)) {
ql_dbg(ql_dbg_tgt_mgt, vha, 0xf080,
"port changed FC ID, %8phC"
" old %x:%x:%x (loop_id 0x%04x)-> new %x:%x:%x\n",
fcport->d_id.b24 = new_fcport->d_id.b24;
fcport->flags |= FCF_LOGIN_NEEDED;
- if (fcport->loop_id != FC_NO_LOOP_ID &&
- (fcport->flags & FCF_FCP2_DEVICE) == 0 &&
- (fcport->flags & FCF_ASYNC_SENT) == 0 &&
- fcport->port_type != FCT_INITIATOR &&
- fcport->port_type != FCT_BROADCAST) {
- ha->isp_ops->fabric_logout(vha, fcport->loop_id,
- fcport->d_id.b.domain, fcport->d_id.b.area,
- fcport->d_id.b.al_pa);
- qla2x00_clear_loop_id(fcport);
- }
-
break;
}
- if (found)
+ if (found) {
+ spin_unlock_irqrestore(&vha->hw->tgt.sess_lock, flags);
continue;
+ }
/* If device was not in our fcports list, then add it. */
new_fcport->scan_state = QLA_FCPORT_FOUND;
- list_add_tail(&new_fcport->list, new_fcports);
+ list_add_tail(&new_fcport->list, &vha->vp_fcports);
+
+ spin_unlock_irqrestore(&vha->hw->tgt.sess_lock, flags);
+
/* Allocate a new replacement fcport. */
nxt_d_id.b24 = new_fcport->d_id.b24;
new_fcport->d_id.b24 = nxt_d_id.b24;
}
- kfree(new_fcport);
+ qla2x00_free_fcport(new_fcport);
+
+ /*
+ * Logout all previous fabric dev marked lost, except FCP2 devices.
+ */
+ list_for_each_entry(fcport, &vha->vp_fcports, list) {
+ if (test_bit(LOOP_RESYNC_NEEDED, &vha->dpc_flags))
+ break;
+
+ if ((fcport->flags & FCF_FABRIC_DEVICE) == 0 ||
+ (fcport->flags & FCF_LOGIN_NEEDED) == 0)
+ continue;
+
+ if (fcport->scan_state == QLA_FCPORT_SCAN) {
+ if ((qla_dual_mode_enabled(vha) ||
+ qla_ini_mode_enabled(vha)) &&
+ atomic_read(&fcport->state) == FCS_ONLINE) {
+ qla2x00_mark_device_lost(vha, fcport,
+ ql2xplogiabsentdevice, 0);
+ if (fcport->loop_id != FC_NO_LOOP_ID &&
+ (fcport->flags & FCF_FCP2_DEVICE) == 0 &&
+ fcport->port_type != FCT_INITIATOR &&
+ fcport->port_type != FCT_BROADCAST) {
+ ql_dbg(ql_dbg_disc, vha, 0xffff,
+ "%s %d %8phC post del sess\n",
+ __func__, __LINE__,
+ fcport->port_name);
+
+ qlt_schedule_sess_for_deletion_lock
+ (fcport);
+ continue;
+ }
+ }
+ }
+ if (fcport->scan_state == QLA_FCPORT_FOUND)
+ qla24xx_fcport_handle_login(vha, fcport);
+ }
return (rval);
}
return (rval);
}
-/*
- * qla2x00_fabric_dev_login
- * Login fabric target device and update FC port database.
- *
- * Input:
- * ha: adapter state pointer.
- * fcport: port structure list pointer.
- * next_loopid: contains value of a new loop ID that can be used
- * by the next login attempt.
- *
- * Returns:
- * qla2x00 local function return status code.
- *
- * Context:
- * Kernel context.
- */
-static int
-qla2x00_fabric_dev_login(scsi_qla_host_t *vha, fc_port_t *fcport,
- uint16_t *next_loopid)
-{
- int rval;
- uint8_t opts;
- struct qla_hw_data *ha = vha->hw;
-
- rval = QLA_SUCCESS;
-
- if (IS_ALOGIO_CAPABLE(ha)) {
- if (fcport->flags & FCF_ASYNC_SENT)
- return rval;
- fcport->flags |= FCF_ASYNC_SENT;
- rval = qla2x00_post_async_login_work(vha, fcport, NULL);
- if (!rval)
- return rval;
- }
-
- fcport->flags &= ~FCF_ASYNC_SENT;
- rval = qla2x00_fabric_login(vha, fcport, next_loopid);
- if (rval == QLA_SUCCESS) {
- /* Send an ADISC to FCP2 devices.*/
- opts = 0;
- if (fcport->flags & FCF_FCP2_DEVICE)
- opts |= BIT_1;
- rval = qla2x00_get_port_database(vha, fcport, opts);
- if (rval != QLA_SUCCESS) {
- ha->isp_ops->fabric_logout(vha, fcport->loop_id,
- fcport->d_id.b.domain, fcport->d_id.b.area,
- fcport->d_id.b.al_pa);
- qla2x00_mark_device_lost(vha, fcport, 1, 0);
- } else {
- qla2x00_update_fcport(vha, fcport);
- }
- } else {
- /* Retry Login. */
- qla2x00_mark_device_lost(vha, fcport, 1, 0);
- }
-
- return (rval);
-}
/*
* qla2x00_fabric_login
spin_unlock_irqrestore(&ha->vport_slock, flags);
qla2x00_rport_del(fcport);
- /*
- * Release the target mode FC NEXUS in
- * qla_target.c, if target mod is enabled.
- */
- qlt_fc_port_deleted(vha, fcport,
- base_vha->total_fcport_update_gen);
-
spin_lock_irqsave(&ha->vport_slock, flags);
}
}
if (!(IS_P3P_TYPE(ha)))
ha->isp_ops->reset_chip(vha);
+ ha->chip_reset++;
+
atomic_set(&vha->loop_down_timer, LOOP_DOWN_TIME);
if (atomic_read(&vha->loop_state) != LOOP_DOWN) {
atomic_set(&vha->loop_state, LOOP_DOWN);
/* Requeue all commands in outstanding command list. */
qla2x00_abort_all_cmds(vha, DID_RESET << 16);
}
-
- ha->chip_reset++;
/* memory barrier */
wmb();
}
if (!status) {
/* Issue a marker after FW becomes ready. */
qla2x00_marker(vha, req, rsp, 0, 0, MK_SYNC_ALL);
-
set_bit(LOOP_RESYNC_NEEDED, &vha->dpc_flags);
}
rval = 1;
}
- if (!qla_ini_mode_enabled(vha)) {
+ if (qla_tgt_mode_enabled(vha)) {
/* Don't enable full login after initial LIP */
nv->firmware_options_1 &= cpu_to_le32(~BIT_13);
/* Don't enable LIP full login for initiator */
for (chksum = 0; cnt--; wptr++)
chksum += le32_to_cpu(*wptr);
+
if (chksum) {
ql_dbg(ql_dbg_init, vha, 0x018c,
"Checksum validation failed for primary image (0x%x)\n",
vha->flags.process_response_queue = 1;
}
+ /* enable RIDA Format2 */
+ if (qla_tgt_mode_enabled(vha) || qla_dual_mode_enabled(vha))
+ icb->firmware_options_3 |= BIT_0;
+
if (rval) {
ql_log(ql_log_warn, vha, 0x0076,
"NVRAM configuration failed.\n");
__func__, ha->fw_options[2]);
}
- if (!ql2xetsenable)
- goto out;
+ /* Move PUREX, ABTS RX & RIDA to ATIOQ */
+ if (ql2xmvasynctoatio) {
+ if (qla_tgt_mode_enabled(vha) ||
+ qla_dual_mode_enabled(vha))
+ ha->fw_options[2] |= BIT_11;
+ else
+ ha->fw_options[2] &= ~BIT_11;
+ }
+
+ if (ql2xetsenable) {
+ /* Enable ETS Burst. */
+ memset(ha->fw_options, 0, sizeof(ha->fw_options));
+ ha->fw_options[2] |= BIT_9;
+ }
+
+ ql_dbg(ql_dbg_init, vha, 0xffff,
+ "%s, add FW options 1-3 = 0x%04x 0x%04x 0x%04x mode %x\n",
+ __func__, ha->fw_options[1], ha->fw_options[2],
+ ha->fw_options[3], vha->host->active_mode);
- /* Enable ETS Burst. */
- memset(ha->fw_options, 0, sizeof(ha->fw_options));
- ha->fw_options[2] |= BIT_9;
-out:
qla2x00_set_fw_options(vha, ha->fw_options);
}
memset(qpair, 0, sizeof(struct qla_qpair));
qpair->hw = vha->hw;
+ qpair->vha = vha;
/* Assign available que pair id */
mutex_lock(&ha->mq_lock);
/* Don't print state transitions during initial allocation of fcport */
if (old_state && old_state != state) {
ql_dbg(ql_dbg_disc, fcport->vha, 0x207d,
- "FCPort state transitioned from %s to %s - "
- "portid=%02x%02x%02x.\n",
+ "FCPort %8phC state transitioned from %s to %s - "
+ "portid=%02x%02x%02x.\n", fcport->port_name,
port_state_str[old_state], port_state_str[state],
fcport->d_id.b.domain, fcport->d_id.b.area,
fcport->d_id.b.al_pa);
memset(sp, 0, sizeof(*sp));
sp->fcport = fcport;
sp->iocbs = 1;
+ sp->vha = qpair->vha;
done:
if (!sp)
QLA_QPAIR_MARK_NOT_BUSY(qpair);
qla2x00_get_sp(scsi_qla_host_t *vha, fc_port_t *fcport, gfp_t flag)
{
srb_t *sp = NULL;
- struct qla_hw_data *ha = vha->hw;
uint8_t bail;
QLA_VHA_MARK_BUSY(vha, bail);
if (unlikely(bail))
return NULL;
- sp = mempool_alloc(ha->srb_mempool, flag);
+ sp = mempool_alloc(vha->hw->srb_mempool, flag);
if (!sp)
goto done;
memset(sp, 0, sizeof(*sp));
sp->fcport = fcport;
sp->iocbs = 1;
+ sp->vha = vha;
done:
if (!sp)
QLA_VHA_MARK_NOT_BUSY(vha);
}
static inline void
-qla2x00_rel_sp(scsi_qla_host_t *vha, srb_t *sp)
+qla2x00_rel_sp(srb_t *sp)
{
- mempool_free(sp, vha->hw->srb_mempool);
- QLA_VHA_MARK_NOT_BUSY(vha);
+ QLA_VHA_MARK_NOT_BUSY(sp->vha);
+ mempool_free(sp, sp->vha->hw->srb_mempool);
}
static inline void
sp->u.iocb_cmd.timer.function = qla2x00_sp_timeout;
add_timer(&sp->u.iocb_cmd.timer);
sp->free = qla2x00_sp_free;
- if ((IS_QLAFX00(sp->fcport->vha->hw)) &&
- (sp->type == SRB_FXIOCB_DCMD))
+ if (IS_QLAFX00(sp->vha->hw) && (sp->type == SRB_FXIOCB_DCMD))
init_completion(&sp->u.iocb_cmd.u.fxiocb.fxiocb_comp);
if (sp->type == SRB_ELS_DCMD)
init_completion(&sp->u.iocb_cmd.u.els_logo.comp);
{
uint16_t cflags;
struct scsi_cmnd *cmd = GET_CMD_SP(sp);
- struct scsi_qla_host *vha = sp->fcport->vha;
+ struct scsi_qla_host *vha = sp->vha;
cflags = 0;
return;
}
- vha = sp->fcport->vha;
+ vha = sp->vha;
cmd_pkt->control_flags |= cpu_to_le16(qla2x00_get_cmd_direction(sp));
/* Three DSDs are available in the Command Type 2 IOCB */
return;
}
- vha = sp->fcport->vha;
+ vha = sp->vha;
cmd_pkt->control_flags |= cpu_to_le16(qla2x00_get_cmd_direction(sp));
/* Two DSDs are available in the Command Type 3 IOCB */
struct rsp_que *rsp;
/* Setup device pointers. */
- vha = sp->fcport->vha;
+ vha = sp->vha;
ha = vha->hw;
reg = &ha->iobase->isp;
cmd = GET_CMD_SP(sp);
return 0;
}
- vha = sp->fcport->vha;
+ vha = sp->vha;
ha = vha->hw;
/* Set transfer direction */
return;
}
- vha = sp->fcport->vha;
+ vha = sp->vha;
/* Set transfer direction */
if (cmd->sc_data_direction == DMA_TO_DEVICE) {
if (sp) {
cmd = GET_CMD_SP(sp);
sgl = scsi_prot_sglist(cmd);
- vha = sp->fcport->vha;
+ vha = sp->vha;
} else if (tc) {
vha = tc->vha;
sgl = tc->prot_sg;
/* Update entry type to indicate Command Type CRC_2 IOCB */
*((uint32_t *)(&cmd_pkt->entry_type)) = cpu_to_le32(COMMAND_TYPE_CRC_2);
- vha = sp->fcport->vha;
+ vha = sp->vha;
ha = vha->hw;
/* No data transfer */
return QLA_SUCCESS;
}
- cmd_pkt->vp_index = sp->fcport->vha->vp_idx;
+ cmd_pkt->vp_index = sp->vha->vp_idx;
/* Set transfer direction */
if (cmd->sc_data_direction == DMA_TO_DEVICE) {
struct req_que *req = NULL;
struct rsp_que *rsp = NULL;
struct scsi_cmnd *cmd = GET_CMD_SP(sp);
- struct scsi_qla_host *vha = sp->fcport->vha;
+ struct scsi_qla_host *vha = sp->vha;
struct qla_hw_data *ha = vha->hw;
/* Setup device pointers. */
cmd_pkt->port_id[0] = sp->fcport->d_id.b.al_pa;
cmd_pkt->port_id[1] = sp->fcport->d_id.b.area;
cmd_pkt->port_id[2] = sp->fcport->d_id.b.domain;
- cmd_pkt->vp_index = sp->fcport->vha->vp_idx;
+ cmd_pkt->vp_index = sp->vha->vp_idx;
int_to_scsilun(cmd->device->lun, &cmd_pkt->lun);
host_to_fcp_swap((uint8_t *)&cmd_pkt->lun, sizeof(cmd_pkt->lun));
struct req_que *req = NULL;
struct rsp_que *rsp = NULL;
struct scsi_cmnd *cmd = GET_CMD_SP(sp);
- struct scsi_qla_host *vha = sp->fcport->vha;
+ struct scsi_qla_host *vha = sp->vha;
struct qla_hw_data *ha = vha->hw;
struct cmd_type_crc_2 *cmd_pkt;
uint32_t status = 0;
logio->port_id[0] = sp->fcport->d_id.b.al_pa;
logio->port_id[1] = sp->fcport->d_id.b.area;
logio->port_id[2] = sp->fcport->d_id.b.domain;
- logio->vp_index = sp->fcport->vha->vp_idx;
+ logio->vp_index = sp->vha->vp_idx;
}
static void
qla2x00_login_iocb(srb_t *sp, struct mbx_entry *mbx)
{
- struct qla_hw_data *ha = sp->fcport->vha->hw;
+ struct qla_hw_data *ha = sp->vha->hw;
struct srb_iocb *lio = &sp->u.iocb_cmd;
uint16_t opts;
mbx->mb2 = cpu_to_le16(sp->fcport->d_id.b.domain);
mbx->mb3 = cpu_to_le16(sp->fcport->d_id.b.area << 8 |
sp->fcport->d_id.b.al_pa);
- mbx->mb9 = cpu_to_le16(sp->fcport->vha->vp_idx);
+ mbx->mb9 = cpu_to_le16(sp->vha->vp_idx);
}
static void
logio->entry_type = LOGINOUT_PORT_IOCB_TYPE;
logio->control_flags =
cpu_to_le16(LCF_COMMAND_LOGO|LCF_IMPL_LOGO);
- if (!sp->fcport->tgt_session ||
- !sp->fcport->tgt_session->keep_nport_handle)
+ if (!sp->fcport->se_sess ||
+ !sp->fcport->keep_nport_handle)
logio->control_flags |= cpu_to_le16(LCF_FREE_NPORT);
logio->nport_handle = cpu_to_le16(sp->fcport->loop_id);
logio->port_id[0] = sp->fcport->d_id.b.al_pa;
logio->port_id[1] = sp->fcport->d_id.b.area;
logio->port_id[2] = sp->fcport->d_id.b.domain;
- logio->vp_index = sp->fcport->vha->vp_idx;
+ logio->vp_index = sp->vha->vp_idx;
}
static void
qla2x00_logout_iocb(srb_t *sp, struct mbx_entry *mbx)
{
- struct qla_hw_data *ha = sp->fcport->vha->hw;
+ struct qla_hw_data *ha = sp->vha->hw;
mbx->entry_type = MBX_IOCB_TYPE;
SET_TARGET_ID(ha, mbx->loop_id, sp->fcport->loop_id);
mbx->mb2 = cpu_to_le16(sp->fcport->d_id.b.domain);
mbx->mb3 = cpu_to_le16(sp->fcport->d_id.b.area << 8 |
sp->fcport->d_id.b.al_pa);
- mbx->mb9 = cpu_to_le16(sp->fcport->vha->vp_idx);
+ mbx->mb9 = cpu_to_le16(sp->vha->vp_idx);
/* Implicit: mbx->mbx10 = 0. */
}
logio->entry_type = LOGINOUT_PORT_IOCB_TYPE;
logio->control_flags = cpu_to_le16(LCF_COMMAND_ADISC);
logio->nport_handle = cpu_to_le16(sp->fcport->loop_id);
- logio->vp_index = sp->fcport->vha->vp_idx;
+ logio->vp_index = sp->vha->vp_idx;
}
static void
qla2x00_adisc_iocb(srb_t *sp, struct mbx_entry *mbx)
{
- struct qla_hw_data *ha = sp->fcport->vha->hw;
+ struct qla_hw_data *ha = sp->vha->hw;
mbx->entry_type = MBX_IOCB_TYPE;
SET_TARGET_ID(ha, mbx->loop_id, sp->fcport->loop_id);
mbx->mb3 = cpu_to_le16(LSW(ha->async_pd_dma));
mbx->mb6 = cpu_to_le16(MSW(MSD(ha->async_pd_dma)));
mbx->mb7 = cpu_to_le16(LSW(MSD(ha->async_pd_dma)));
- mbx->mb9 = cpu_to_le16(sp->fcport->vha->vp_idx);
+ mbx->mb9 = cpu_to_le16(sp->vha->vp_idx);
}
static void
}
static void
-qla2x00_els_dcmd_sp_free(void *ptr, void *data)
+qla2x00_els_dcmd_sp_free(void *data)
{
- struct scsi_qla_host *vha = (scsi_qla_host_t *)ptr;
- struct qla_hw_data *ha = vha->hw;
- srb_t *sp = (srb_t *)data;
+ srb_t *sp = data;
struct srb_iocb *elsio = &sp->u.iocb_cmd;
kfree(sp->fcport);
if (elsio->u.els_logo.els_logo_pyld)
- dma_free_coherent(&ha->pdev->dev, DMA_POOL_SIZE,
+ dma_free_coherent(&sp->vha->hw->pdev->dev, DMA_POOL_SIZE,
elsio->u.els_logo.els_logo_pyld,
elsio->u.els_logo.els_logo_pyld_dma);
del_timer(&elsio->timer);
- qla2x00_rel_sp(vha, sp);
+ qla2x00_rel_sp(sp);
}
static void
qla2x00_els_dcmd_iocb_timeout(void *data)
{
- srb_t *sp = (srb_t *)data;
- struct srb_iocb *lio = &sp->u.iocb_cmd;
+ srb_t *sp = data;
fc_port_t *fcport = sp->fcport;
- struct scsi_qla_host *vha = fcport->vha;
+ struct scsi_qla_host *vha = sp->vha;
struct qla_hw_data *ha = vha->hw;
+ struct srb_iocb *lio = &sp->u.iocb_cmd;
unsigned long flags = 0;
ql_dbg(ql_dbg_io, vha, 0x3069,
}
static void
-qla2x00_els_dcmd_sp_done(void *data, void *ptr, int res)
+qla2x00_els_dcmd_sp_done(void *ptr, int res)
{
- srb_t *sp = (srb_t *)ptr;
+ srb_t *sp = ptr;
fc_port_t *fcport = sp->fcport;
struct srb_iocb *lio = &sp->u.iocb_cmd;
- struct scsi_qla_host *vha = fcport->vha;
+ struct scsi_qla_host *vha = sp->vha;
ql_dbg(ql_dbg_io, vha, 0x3072,
"%s hdl=%x, portid=%02x%02x%02x done\n",
GFP_KERNEL);
if (!elsio->u.els_logo.els_logo_pyld) {
- sp->free(vha, sp);
+ sp->free(sp);
return QLA_FUNCTION_FAILED;
}
rval = qla2x00_start_sp(sp);
if (rval != QLA_SUCCESS) {
- sp->free(vha, sp);
+ sp->free(sp);
return QLA_FUNCTION_FAILED;
}
wait_for_completion(&elsio->u.els_logo.comp);
- sp->free(vha, sp);
+ sp->free(sp);
return rval;
}
static void
qla24xx_els_logo_iocb(srb_t *sp, struct els_entry_24xx *els_iocb)
{
- scsi_qla_host_t *vha = sp->fcport->vha;
+ scsi_qla_host_t *vha = sp->vha;
struct srb_iocb *elsio = &sp->u.iocb_cmd;
els_iocb->entry_type = ELS_IOCB_TYPE;
els_iocb->rx_address[1] = 0;
els_iocb->rx_len = 0;
- sp->fcport->vha->qla_stats.control_requests++;
+ sp->vha->qla_stats.control_requests++;
}
static void
els_iocb->handle = sp->handle;
els_iocb->nport_handle = cpu_to_le16(sp->fcport->loop_id);
els_iocb->tx_dsd_count = cpu_to_le16(bsg_job->request_payload.sg_cnt);
- els_iocb->vp_index = sp->fcport->vha->vp_idx;
+ els_iocb->vp_index = sp->vha->vp_idx;
els_iocb->sof_type = EST_SOFI3;
els_iocb->rx_dsd_count = cpu_to_le16(bsg_job->reply_payload.sg_cnt);
els_iocb->rx_len = cpu_to_le32(sg_dma_len
(bsg_job->reply_payload.sg_list));
- sp->fcport->vha->qla_stats.control_requests++;
+ sp->vha->qla_stats.control_requests++;
}
static void
struct scatterlist *sg;
int index;
uint16_t tot_dsds;
- scsi_qla_host_t *vha = sp->fcport->vha;
+ scsi_qla_host_t *vha = sp->vha;
struct qla_hw_data *ha = vha->hw;
struct bsg_job *bsg_job = sp->u.bsg_job;
int loop_iterartion = 0;
}
ct_iocb->entry_count = entry_count;
- sp->fcport->vha->qla_stats.control_requests++;
+ sp->vha->qla_stats.control_requests++;
}
static void
struct scatterlist *sg;
int index;
uint16_t tot_dsds;
- scsi_qla_host_t *vha = sp->fcport->vha;
+ scsi_qla_host_t *vha = sp->vha;
struct qla_hw_data *ha = vha->hw;
struct bsg_job *bsg_job = sp->u.bsg_job;
int loop_iterartion = 0;
ct_iocb->handle = sp->handle;
ct_iocb->nport_handle = cpu_to_le16(sp->fcport->loop_id);
- ct_iocb->vp_index = sp->fcport->vha->vp_idx;
+ ct_iocb->vp_index = sp->vha->vp_idx;
ct_iocb->comp_status = cpu_to_le16(0);
ct_iocb->cmd_dsd_count =
uint32_t *fcp_dl;
uint8_t additional_cdb_len;
struct ct6_dsd *ctx;
- struct scsi_qla_host *vha = sp->fcport->vha;
+ struct scsi_qla_host *vha = sp->vha;
struct qla_hw_data *ha = vha->hw;
struct req_que *req = NULL;
struct rsp_que *rsp = NULL;
cmd_pkt->port_id[0] = sp->fcport->d_id.b.al_pa;
cmd_pkt->port_id[1] = sp->fcport->d_id.b.area;
cmd_pkt->port_id[2] = sp->fcport->d_id.b.domain;
- cmd_pkt->vp_index = sp->fcport->vha->vp_idx;
+ cmd_pkt->vp_index = sp->vha->vp_idx;
/* Build IOCB segments */
if (qla24xx_build_scsi_type_6_iocbs(sp, cmd_pkt, tot_dsds))
cmd_pkt->port_id[0] = sp->fcport->d_id.b.al_pa;
cmd_pkt->port_id[1] = sp->fcport->d_id.b.area;
cmd_pkt->port_id[2] = sp->fcport->d_id.b.domain;
- cmd_pkt->vp_index = sp->fcport->vha->vp_idx;
+ cmd_pkt->vp_index = sp->vha->vp_idx;
int_to_scsilun(cmd->device->lun, &cmd_pkt->lun);
host_to_fcp_swap((uint8_t *)&cmd_pkt->lun,
qla24xx_abort_iocb(srb_t *sp, struct abort_entry_24xx *abt_iocb)
{
struct srb_iocb *aio = &sp->u.iocb_cmd;
- scsi_qla_host_t *vha = sp->fcport->vha;
+ scsi_qla_host_t *vha = sp->vha;
struct req_que *req = vha->req;
memset(abt_iocb, 0, sizeof(struct abort_entry_24xx));
wmb();
}
+static void
+qla2x00_mb_iocb(srb_t *sp, struct mbx_24xx_entry *mbx)
+{
+ int i, sz;
+
+ mbx->entry_type = MBX_IOCB_TYPE;
+ mbx->handle = sp->handle;
+ sz = min(ARRAY_SIZE(mbx->mb), ARRAY_SIZE(sp->u.iocb_cmd.u.mbx.out_mb));
+
+ for (i = 0; i < sz; i++)
+ mbx->mb[i] = cpu_to_le16(sp->u.iocb_cmd.u.mbx.out_mb[i]);
+}
+
+static void
+qla2x00_ctpthru_cmd_iocb(srb_t *sp, struct ct_entry_24xx *ct_pkt)
+{
+ sp->u.iocb_cmd.u.ctarg.iocb = ct_pkt;
+ qla24xx_prep_ms_iocb(sp->vha, &sp->u.iocb_cmd.u.ctarg);
+ ct_pkt->handle = sp->handle;
+}
+
+static void qla2x00_send_notify_ack_iocb(srb_t *sp,
+ struct nack_to_isp *nack)
+{
+ struct imm_ntfy_from_isp *ntfy = sp->u.iocb_cmd.u.nack.ntfy;
+
+ nack->entry_type = NOTIFY_ACK_TYPE;
+ nack->entry_count = 1;
+ nack->ox_id = ntfy->ox_id;
+
+ nack->u.isp24.handle = sp->handle;
+ nack->u.isp24.nport_handle = ntfy->u.isp24.nport_handle;
+ if (le16_to_cpu(ntfy->u.isp24.status) == IMM_NTFY_ELS) {
+ nack->u.isp24.flags = ntfy->u.isp24.flags &
+ cpu_to_le32(NOTIFY24XX_FLAGS_PUREX_IOCB);
+ }
+ nack->u.isp24.srr_rx_id = ntfy->u.isp24.srr_rx_id;
+ nack->u.isp24.status = ntfy->u.isp24.status;
+ nack->u.isp24.status_subcode = ntfy->u.isp24.status_subcode;
+ nack->u.isp24.fw_handle = ntfy->u.isp24.fw_handle;
+ nack->u.isp24.exchange_address = ntfy->u.isp24.exchange_address;
+ nack->u.isp24.srr_rel_offs = ntfy->u.isp24.srr_rel_offs;
+ nack->u.isp24.srr_ui = ntfy->u.isp24.srr_ui;
+ nack->u.isp24.srr_flags = 0;
+ nack->u.isp24.srr_reject_code = 0;
+ nack->u.isp24.srr_reject_code_expl = 0;
+ nack->u.isp24.vp_index = ntfy->u.isp24.vp_index;
+}
+
int
qla2x00_start_sp(srb_t *sp)
{
int rval;
- struct qla_hw_data *ha = sp->fcport->vha->hw;
+ scsi_qla_host_t *vha = sp->vha;
+ struct qla_hw_data *ha = vha->hw;
void *pkt;
unsigned long flags;
rval = QLA_FUNCTION_FAILED;
spin_lock_irqsave(&ha->hardware_lock, flags);
- pkt = qla2x00_alloc_iocbs(sp->fcport->vha, sp);
+ pkt = qla2x00_alloc_iocbs(vha, sp);
if (!pkt) {
- ql_log(ql_log_warn, sp->fcport->vha, 0x700c,
+ ql_log(ql_log_warn, vha, 0x700c,
"qla2x00_alloc_iocbs failed.\n");
goto done;
}
case SRB_ELS_DCMD:
qla24xx_els_logo_iocb(sp, pkt);
break;
+ case SRB_CT_PTHRU_CMD:
+ qla2x00_ctpthru_cmd_iocb(sp, pkt);
+ break;
+ case SRB_MB_IOCB:
+ qla2x00_mb_iocb(sp, pkt);
+ break;
+ case SRB_NACK_PLOGI:
+ case SRB_NACK_PRLI:
+ case SRB_NACK_LOGO:
+ qla2x00_send_notify_ack_iocb(sp, pkt);
+ break;
default:
break;
}
wmb();
- qla2x00_start_iocbs(sp->fcport->vha, ha->req_q_map[0]);
+ qla2x00_start_iocbs(vha, ha->req_q_map[0]);
done:
spin_unlock_irqrestore(&ha->hardware_lock, flags);
return rval;
return ret;
}
-static inline fc_port_t *
+fc_port_t *
qla2x00_find_fcport_by_loopid(scsi_qla_host_t *vha, uint16_t loop_id)
{
- fc_port_t *fcport;
+ fc_port_t *f, *tf;
+
+ f = tf = NULL;
+ list_for_each_entry_safe(f, tf, &vha->vp_fcports, list)
+ if (f->loop_id == loop_id)
+ return f;
+ return NULL;
+}
+
+fc_port_t *
+qla2x00_find_fcport_by_wwpn(scsi_qla_host_t *vha, u8 *wwpn, u8 incl_deleted)
+{
+ fc_port_t *f, *tf;
+
+ f = tf = NULL;
+ list_for_each_entry_safe(f, tf, &vha->vp_fcports, list) {
+ if (memcmp(f->port_name, wwpn, WWN_SIZE) == 0) {
+ if (incl_deleted)
+ return f;
+ else if (f->deleted == 0)
+ return f;
+ }
+ }
+ return NULL;
+}
- list_for_each_entry(fcport, &vha->vp_fcports, list)
- if (fcport->loop_id == loop_id)
- return fcport;
+fc_port_t *
+qla2x00_find_fcport_by_nportid(scsi_qla_host_t *vha, port_id_t *id,
+ u8 incl_deleted)
+{
+ fc_port_t *f, *tf;
+
+ f = tf = NULL;
+ list_for_each_entry_safe(f, tf, &vha->vp_fcports, list) {
+ if (f->d_id.b24 == id->b24) {
+ if (incl_deleted)
+ return f;
+ else if (f->deleted == 0)
+ return f;
+ }
+ }
return NULL;
}
ql_dbg(ql_dbg_async, vha, 0x508a,
"Marking port lost loopid=%04x portid=%06x.\n",
fcport->loop_id, fcport->d_id.b24);
- qla2x00_mark_device_lost(fcport->vha, fcport, 1, 1);
+ if (qla_ini_mode_enabled(vha)) {
+ qla2x00_mark_device_lost(fcport->vha, fcport, 1, 1);
+ fcport->logout_on_delete = 0;
+ qlt_schedule_sess_for_deletion_lock(fcport);
+ }
break;
global_port_update:
qla2x00_mark_all_devices_lost(vha, 1);
- if (vha->vp_idx == 0 && !qla_ini_mode_enabled(vha))
- set_bit(SCR_PENDING, &vha->dpc_flags);
-
set_bit(LOOP_RESYNC_NEEDED, &vha->dpc_flags);
set_bit(LOCAL_LOOP_UPDATE, &vha->dpc_flags);
set_bit(VP_CONFIG_OK, &vha->vp_flags);
if (qla2x00_is_a_vp_did(vha, rscn_entry))
break;
- /*
- * Search for the rport related to this RSCN entry and mark it
- * as lost.
- */
- list_for_each_entry(fcport, &vha->vp_fcports, list) {
- if (atomic_read(&fcport->state) != FCS_ONLINE)
- continue;
- if (fcport->d_id.b24 == rscn_entry) {
- qla2x00_mark_device_lost(vha, fcport, 0, 0);
- break;
- }
- }
-
atomic_set(&vha->loop_down_timer, 0);
vha->flags.management_server_logged_in = 0;
-
- set_bit(LOOP_RESYNC_NEEDED, &vha->dpc_flags);
- set_bit(RSCN_UPDATE, &vha->dpc_flags);
- qla2x00_post_aen_work(vha, FCH_EVT_RSCN, rscn_entry);
+ {
+ struct event_arg ea;
+
+ memset(&ea, 0, sizeof(ea));
+ ea.event = FCME_RSCN;
+ ea.id.b24 = rscn_entry;
+ ea.id.b.rsvd_1 = rscn_entry >> 24;
+ qla2x00_fcport_event_handler(vha, &ea);
+ qla2x00_post_aen_work(vha, FCH_EVT_RSCN, rscn_entry);
+ }
break;
-
/* case MBA_RIO_RESPONSE: */
case MBA_ZIO_RESPONSE:
ql_dbg(ql_dbg_async, vha, 0x5015,
req->outstanding_cmds[index] = NULL;
/* Save ISP completion status */
- sp->done(ha, sp, DID_OK << 16);
+ sp->done(sp, DID_OK << 16);
} else {
ql_log(ql_log_warn, vha, 0x3016, "Invalid SCSI SRB.\n");
index = LSW(pkt->handle);
if (index >= req->num_outstanding_cmds) {
ql_log(ql_log_warn, vha, 0x5031,
- "Invalid command index (%x).\n", index);
+ "Invalid command index (%x) type %8ph.\n",
+ index, iocb);
if (IS_P3P_TYPE(ha))
set_bit(FCOE_CTX_RESET_NEEDED, &vha->dpc_flags);
else
le16_to_cpu(mbx->mb7));
logio_done:
- sp->done(vha, sp, 0);
+ sp->done(sp, 0);
}
static void
-qla2x00_ct_entry(scsi_qla_host_t *vha, struct req_que *req,
- sts_entry_t *pkt, int iocb_type)
+qla24xx_mbx_iocb_entry(scsi_qla_host_t *vha, struct req_que *req,
+ struct mbx_24xx_entry *pkt)
{
- const char func[] = "CT_IOCB";
- const char *type;
+ const char func[] = "MBX-IOCB2";
srb_t *sp;
- struct bsg_job *bsg_job;
- struct fc_bsg_reply *bsg_reply;
- uint16_t comp_status;
+ struct srb_iocb *si;
+ u16 sz, i;
int res;
sp = qla2x00_get_sp_from_handle(vha, func, req, pkt);
if (!sp)
return;
- bsg_job = sp->u.bsg_job;
- bsg_reply = bsg_job->reply;
+ si = &sp->u.iocb_cmd;
+ sz = min(ARRAY_SIZE(pkt->mb), ARRAY_SIZE(sp->u.iocb_cmd.u.mbx.in_mb));
- type = "ct pass-through";
+ for (i = 0; i < sz; i++)
+ si->u.mbx.in_mb[i] = le16_to_cpu(pkt->mb[i]);
- comp_status = le16_to_cpu(pkt->comp_status);
+ res = (si->u.mbx.in_mb[0] & MBS_MASK);
- /* return FC_CTELS_STATUS_OK and leave the decoding of the ELS/CT
- * fc payload to the caller
- */
- bsg_reply->reply_data.ctels_reply.status = FC_CTELS_STATUS_OK;
- bsg_job->reply_len = sizeof(struct fc_bsg_reply);
+ sp->done(sp, res);
+}
- if (comp_status != CS_COMPLETE) {
- if (comp_status == CS_DATA_UNDERRUN) {
- res = DID_OK << 16;
- bsg_reply->reply_payload_rcv_len =
- le16_to_cpu(((sts_entry_t *)pkt)->rsp_info_len);
+static void
+qla24xxx_nack_iocb_entry(scsi_qla_host_t *vha, struct req_que *req,
+ struct nack_to_isp *pkt)
+{
+ const char func[] = "nack";
+ srb_t *sp;
+ int res = 0;
- ql_log(ql_log_warn, vha, 0x5048,
- "CT pass-through-%s error "
- "comp_status-status=0x%x total_byte = 0x%x.\n",
- type, comp_status,
- bsg_reply->reply_payload_rcv_len);
- } else {
- ql_log(ql_log_warn, vha, 0x5049,
- "CT pass-through-%s error "
- "comp_status-status=0x%x.\n", type, comp_status);
- res = DID_ERROR << 16;
- bsg_reply->reply_payload_rcv_len = 0;
- }
- ql_dump_buffer(ql_dbg_async + ql_dbg_buffer, vha, 0x5035,
- (uint8_t *)pkt, sizeof(*pkt));
- } else {
- res = DID_OK << 16;
- bsg_reply->reply_payload_rcv_len =
- bsg_job->reply_payload.payload_len;
- bsg_job->reply_len = 0;
- }
+ sp = qla2x00_get_sp_from_handle(vha, func, req, pkt);
+ if (!sp)
+ return;
+
+ if (pkt->u.isp2x.status != cpu_to_le16(NOTIFY_ACK_SUCCESS))
+ res = QLA_FUNCTION_FAILED;
+
+ sp->done(sp, res);
+}
+
+static void
+qla2x00_ct_entry(scsi_qla_host_t *vha, struct req_que *req,
+ sts_entry_t *pkt, int iocb_type)
+{
+ const char func[] = "CT_IOCB";
+ const char *type;
+ srb_t *sp;
+ struct bsg_job *bsg_job;
+ struct fc_bsg_reply *bsg_reply;
+ uint16_t comp_status;
+ int res = 0;
+
+ sp = qla2x00_get_sp_from_handle(vha, func, req, pkt);
+ if (!sp)
+ return;
- sp->done(vha, sp, res);
+ switch (sp->type) {
+ case SRB_CT_CMD:
+ bsg_job = sp->u.bsg_job;
+ bsg_reply = bsg_job->reply;
+
+ type = "ct pass-through";
+
+ comp_status = le16_to_cpu(pkt->comp_status);
+
+ /*
+ * return FC_CTELS_STATUS_OK and leave the decoding of the ELS/CT
+ * fc payload to the caller
+ */
+ bsg_reply->reply_data.ctels_reply.status = FC_CTELS_STATUS_OK;
+ bsg_job->reply_len = sizeof(struct fc_bsg_reply);
+
+ if (comp_status != CS_COMPLETE) {
+ if (comp_status == CS_DATA_UNDERRUN) {
+ res = DID_OK << 16;
+ bsg_reply->reply_payload_rcv_len =
+ le16_to_cpu(((sts_entry_t *)pkt)->rsp_info_len);
+
+ ql_log(ql_log_warn, vha, 0x5048,
+ "CT pass-through-%s error comp_status=0x%x total_byte=0x%x.\n",
+ type, comp_status,
+ bsg_reply->reply_payload_rcv_len);
+ } else {
+ ql_log(ql_log_warn, vha, 0x5049,
+ "CT pass-through-%s error comp_status=0x%x.\n",
+ type, comp_status);
+ res = DID_ERROR << 16;
+ bsg_reply->reply_payload_rcv_len = 0;
+ }
+ ql_dump_buffer(ql_dbg_async + ql_dbg_buffer, vha, 0x5035,
+ (uint8_t *)pkt, sizeof(*pkt));
+ } else {
+ res = DID_OK << 16;
+ bsg_reply->reply_payload_rcv_len =
+ bsg_job->reply_payload.payload_len;
+ bsg_job->reply_len = 0;
+ }
+ break;
+ case SRB_CT_PTHRU_CMD:
+ /*
+ * borrowing sts_entry_24xx.comp_status.
+ * same location as ct_entry_24xx.comp_status
+ */
+ res = qla2x00_chk_ms_status(vha, (ms_iocb_entry_t *)pkt,
+ (struct ct_sns_rsp *)sp->u.iocb_cmd.u.ctarg.rsp,
+ sp->name);
+ break;
+ }
+
+ sp->done(sp, res);
}
static void
type = "Driver ELS logo";
ql_dbg(ql_dbg_user, vha, 0x5047,
"Completing %s: (%p) type=%d.\n", type, sp, sp->type);
- sp->done(vha, sp, 0);
+ sp->done(sp, 0);
+ return;
+ case SRB_CT_PTHRU_CMD:
+ /* borrowing sts_entry_24xx.comp_status.
+ same location as ct_entry_24xx.comp_status
+ */
+ res = qla2x00_chk_ms_status(vha, (ms_iocb_entry_t *)pkt,
+ (struct ct_sns_rsp *)sp->u.iocb_cmd.u.ctarg.rsp,
+ sp->name);
+ sp->done(sp, res);
return;
default:
ql_dbg(ql_dbg_user, vha, 0x503e,
bsg_job->reply_len = 0;
}
- sp->done(vha, sp, res);
+ sp->done(sp, res);
}
static void
fcport->d_id.b.area, fcport->d_id.b.al_pa,
le32_to_cpu(logio->io_parameter[0]));
+ vha->hw->exch_starvation = 0;
data[0] = MBS_COMMAND_COMPLETE;
if (sp->type != SRB_LOGIN_CMD)
goto logio_done;
iop[0] = le32_to_cpu(logio->io_parameter[0]);
iop[1] = le32_to_cpu(logio->io_parameter[1]);
+ lio->u.logio.iop[0] = iop[0];
+ lio->u.logio.iop[1] = iop[1];
switch (iop[0]) {
case LSC_SCODE_PORTID_USED:
data[0] = MBS_PORT_ID_USED;
case LSC_SCODE_NPORT_USED:
data[0] = MBS_LOOP_ID_USED;
break;
+ case LSC_SCODE_NOXCB:
+ vha->hw->exch_starvation++;
+ if (vha->hw->exch_starvation > 5) {
+ ql_log(ql_log_warn, vha, 0xffff,
+ "Exchange starvation. Resetting RISC\n");
+
+ vha->hw->exch_starvation = 0;
+
+ if (IS_P3P_TYPE(vha->hw))
+ set_bit(FCOE_CTX_RESET_NEEDED, &vha->dpc_flags);
+ else
+ set_bit(ISP_ABORT_NEEDED, &vha->dpc_flags);
+ qla2xxx_wake_dpc(vha);
+ }
+ /* drop through */
default:
data[0] = MBS_COMMAND_ERROR;
break;
le32_to_cpu(logio->io_parameter[1]));
logio_done:
- sp->done(vha, sp, 0);
+ sp->done(sp, 0);
}
static void
ql_dump_buffer(ql_dbg_async + ql_dbg_buffer, vha, 0x5055,
(uint8_t *)sts, sizeof(*sts));
- sp->done(vha, sp, 0);
+ sp->done(sp, 0);
}
/**
qla2x00_handle_sense(srb_t *sp, uint8_t *sense_data, uint32_t par_sense_len,
uint32_t sense_len, struct rsp_que *rsp, int res)
{
- struct scsi_qla_host *vha = sp->fcport->vha;
+ struct scsi_qla_host *vha = sp->vha;
struct scsi_cmnd *cp = GET_CMD_SP(sp);
uint32_t track_sense_len;
if (sense_len) {
ql_dbg(ql_dbg_io + ql_dbg_buffer, vha, 0x301c,
"Check condition Sense data, nexus%ld:%d:%llu cmd=%p.\n",
- sp->fcport->vha->host_no, cp->device->id, cp->device->lun,
+ sp->vha->host_no, cp->device->id, cp->device->lun,
cp);
ql_dump_buffer(ql_dbg_io + ql_dbg_buffer, vha, 0x302b,
cp->sense_buffer, sense_len);
static inline int
qla2x00_handle_dif_error(srb_t *sp, struct sts_entry_24xx *sts24)
{
- struct scsi_qla_host *vha = sp->fcport->vha;
+ struct scsi_qla_host *vha = sp->vha;
struct scsi_cmnd *cmd = GET_CMD_SP(sp);
uint8_t *ap = &sts24->data[12];
uint8_t *ep = &sts24->data[20];
bsg_job->reply_len = sizeof(struct fc_bsg_reply);
/* Always return DID_OK, bsg will send the vendor specific response
* in this case only */
- sp->done(vha, sp, (DID_OK << 6));
+ sp->done(sp, DID_OK << 6);
}
int res = 0;
uint16_t state_flags = 0;
uint16_t retry_delay = 0;
+ uint8_t no_logout = 0;
sts = (sts_entry_t *) pkt;
sts24 = (struct sts_entry_24xx *) pkt;
break;
case CS_PORT_LOGGED_OUT:
+ no_logout = 1;
case CS_PORT_CONFIG_CHG:
case CS_PORT_BUSY:
case CS_INCOMPLETE:
break;
}
- ql_dbg(ql_dbg_io, fcport->vha, 0x3021,
- "Port to be marked lost on fcport=%02x%02x%02x, current "
- "port state= %s.\n", fcport->d_id.b.domain,
- fcport->d_id.b.area, fcport->d_id.b.al_pa,
- port_state_str[atomic_read(&fcport->state)]);
+ if (atomic_read(&fcport->state) == FCS_ONLINE) {
+ ql_dbg(ql_dbg_disc, fcport->vha, 0x3021,
+ "Port to be marked lost on fcport=%02x%02x%02x, current "
+ "port state= %s comp_status %x.\n", fcport->d_id.b.domain,
+ fcport->d_id.b.area, fcport->d_id.b.al_pa,
+ port_state_str[atomic_read(&fcport->state)],
+ comp_status);
+
+ if (no_logout)
+ fcport->logout_on_delete = 0;
- if (atomic_read(&fcport->state) == FCS_ONLINE)
qla2x00_mark_device_lost(fcport->vha, fcport, 1, 1);
+ qlt_schedule_sess_for_deletion_lock(fcport);
+ }
+
break;
case CS_ABORTED:
resid_len, fw_resid_len, sp, cp);
if (rsp->status_srb == NULL)
- sp->done(ha, sp, res);
+ sp->done(sp, res);
}
/**
/* Place command on done queue. */
if (sense_len == 0) {
rsp->status_srb = NULL;
- sp->done(ha, sp, cp->result);
+ sp->done(sp, cp->result);
}
}
sp = qla2x00_get_sp_from_handle(vha, func, req, pkt);
if (sp) {
- sp->done(ha, sp, res);
+ sp->done(sp, res);
return;
}
fatal:
abt = &sp->u.iocb_cmd;
abt->u.abt.comp_status = le32_to_cpu(pkt->nport_handle);
- sp->done(vha, sp, 0);
+ sp->done(sp, 0);
}
/**
}
case ABTS_RESP_24XX:
case CTIO_TYPE7:
- case NOTIFY_ACK_TYPE:
case CTIO_CRC2:
qlt_response_pkt_all_vps(vha, (response_t *)pkt);
break;
+ case NOTIFY_ACK_TYPE:
+ if (pkt->handle == QLA_TGT_SKIP_HANDLE)
+ qlt_response_pkt_all_vps(vha, (response_t *)pkt);
+ else
+ qla24xxx_nack_iocb_entry(vha, rsp->req,
+ (struct nack_to_isp *)pkt);
+ break;
case MARKER_TYPE:
/* Do nothing in this case, this check is to prevent it
* from falling into default case
qla24xx_abort_iocb_entry(vha, rsp->req,
(struct abort_entry_24xx *)pkt);
break;
+ case MBX_IOCB_TYPE:
+ qla24xx_mbx_iocb_entry(vha, rsp->req,
+ (struct mbx_24xx_entry *)pkt);
+ break;
default:
/* Type Not Supported. */
ql_dbg(ql_dbg_async, vha, 0x5042,
if (IS_P3P_TYPE(ha)) {
struct device_reg_82xx __iomem *reg = &ha->iobase->isp82;
WRT_REG_DWORD(®->rsp_q_out[0], rsp->ring_index);
- } else
+ } else {
WRT_REG_DWORD(rsp->rsp_q_out, rsp->ring_index);
+ }
}
static void
int i, ret;
struct qla_msix_entry *qentry;
scsi_qla_host_t *vha = pci_get_drvdata(ha->pdev);
+ int min_vecs = QLA_BASE_VECTORS;
struct irq_affinity desc = {
.pre_vectors = QLA_BASE_VECTORS,
};
- if (QLA_TGT_MODE_ENABLED() && IS_ATIO_MSIX_CAPABLE(ha))
+ if (QLA_TGT_MODE_ENABLED() && IS_ATIO_MSIX_CAPABLE(ha)) {
desc.pre_vectors++;
+ min_vecs++;
+ }
- ret = pci_alloc_irq_vectors_affinity(ha->pdev, QLA_BASE_VECTORS,
+ ret = pci_alloc_irq_vectors_affinity(ha->pdev, min_vecs,
ha->msix_count, PCI_IRQ_MSIX | PCI_IRQ_AFFINITY,
&desc);
return rval;
}
-/*
- * qla2x00_get_node_name_list
- * Issue get node name list mailbox command, kmalloc()
- * and return the resulting list. Caller must kfree() it!
- *
- * Input:
- * ha = adapter state pointer.
- * out_data = resulting list
- * out_len = length of the resulting list
- *
- * Returns:
- * qla2x00 local function return status code.
- *
- * Context:
- * Kernel context.
- */
-int
-qla2x00_get_node_name_list(scsi_qla_host_t *vha, void **out_data, int *out_len)
-{
- struct qla_hw_data *ha = vha->hw;
- struct qla_port_24xx_data *list = NULL;
- void *pmap;
- mbx_cmd_t mc;
- dma_addr_t pmap_dma;
- ulong dma_size;
- int rval, left;
-
- left = 1;
- while (left > 0) {
- dma_size = left * sizeof(*list);
- pmap = dma_alloc_coherent(&ha->pdev->dev, dma_size,
- &pmap_dma, GFP_KERNEL);
- if (!pmap) {
- ql_log(ql_log_warn, vha, 0x113f,
- "%s(%ld): DMA Alloc failed of %ld\n",
- __func__, vha->host_no, dma_size);
- rval = QLA_MEMORY_ALLOC_FAILED;
- goto out;
- }
-
- mc.mb[0] = MBC_PORT_NODE_NAME_LIST;
- mc.mb[1] = BIT_1 | BIT_3;
- mc.mb[2] = MSW(pmap_dma);
- mc.mb[3] = LSW(pmap_dma);
- mc.mb[6] = MSW(MSD(pmap_dma));
- mc.mb[7] = LSW(MSD(pmap_dma));
- mc.mb[8] = dma_size;
- mc.out_mb = MBX_0|MBX_1|MBX_2|MBX_3|MBX_6|MBX_7|MBX_8;
- mc.in_mb = MBX_0|MBX_1;
- mc.tov = 30;
- mc.flags = MBX_DMA_IN;
-
- rval = qla2x00_mailbox_command(vha, &mc);
- if (rval != QLA_SUCCESS) {
- if ((mc.mb[0] == MBS_COMMAND_ERROR) &&
- (mc.mb[1] == 0xA)) {
- left += le16_to_cpu(mc.mb[2]) /
- sizeof(struct qla_port_24xx_data);
- goto restart;
- }
- goto out_free;
- }
-
- left = 0;
-
- list = kmemdup(pmap, dma_size, GFP_KERNEL);
- if (!list) {
- ql_log(ql_log_warn, vha, 0x1140,
- "%s(%ld): failed to allocate node names list "
- "structure.\n", __func__, vha->host_no);
- rval = QLA_MEMORY_ALLOC_FAILED;
- goto out_free;
- }
-
-restart:
- dma_free_coherent(&ha->pdev->dev, dma_size, pmap, pmap_dma);
- }
-
- *out_data = list;
- *out_len = dma_size;
-
-out:
- return rval;
-
-out_free:
- dma_free_coherent(&ha->pdev->dev, dma_size, pmap, pmap_dma);
- return rval;
-}
/*
* qla2x00_get_port_database
qla24xx_report_id_acquisition(scsi_qla_host_t *vha,
struct vp_rpt_id_entry_24xx *rptid_entry)
{
- uint8_t vp_idx;
- uint16_t stat = le16_to_cpu(rptid_entry->vp_idx);
struct qla_hw_data *ha = vha->hw;
- scsi_qla_host_t *vp;
+ scsi_qla_host_t *vp = NULL;
unsigned long flags;
int found;
return;
if (rptid_entry->format == 0) {
+ /* loop */
ql_dbg(ql_dbg_mbx + ql_dbg_verbose, vha, 0x10b7,
"Format 0 : Number of VPs setup %d, number of "
- "VPs acquired %d.\n",
- MSB(le16_to_cpu(rptid_entry->vp_count)),
- LSB(le16_to_cpu(rptid_entry->vp_count)));
+ "VPs acquired %d.\n", rptid_entry->vp_setup,
+ rptid_entry->vp_acquired);
ql_dbg(ql_dbg_mbx + ql_dbg_verbose, vha, 0x10b8,
"Primary port id %02x%02x%02x.\n",
rptid_entry->port_id[2], rptid_entry->port_id[1],
rptid_entry->port_id[0]);
+
+ vha->d_id.b.domain = rptid_entry->port_id[2];
+ vha->d_id.b.area = rptid_entry->port_id[1];
+ vha->d_id.b.al_pa = rptid_entry->port_id[0];
+
+ spin_lock_irqsave(&ha->vport_slock, flags);
+ qlt_update_vp_map(vha, SET_AL_PA);
+ spin_unlock_irqrestore(&ha->vport_slock, flags);
+
} else if (rptid_entry->format == 1) {
- vp_idx = LSB(stat);
+ /* fabric */
ql_dbg(ql_dbg_mbx + ql_dbg_verbose, vha, 0x10b9,
"Format 1: VP[%d] enabled - status %d - with "
- "port id %02x%02x%02x.\n", vp_idx, MSB(stat),
+ "port id %02x%02x%02x.\n", rptid_entry->vp_idx,
+ rptid_entry->vp_status,
rptid_entry->port_id[2], rptid_entry->port_id[1],
rptid_entry->port_id[0]);
/* buffer to buffer credit flag */
- vha->flags.bbcr_enable = (rptid_entry->bbcr & 0xf) != 0;
-
- /* FA-WWN is only for physical port */
- if (!vp_idx) {
- void *wwpn = ha->init_cb->port_name;
+ vha->flags.bbcr_enable = (rptid_entry->u.f1.bbcr & 0xf) != 0;
+
+ if (rptid_entry->vp_idx == 0) {
+ if (rptid_entry->vp_status == VP_STAT_COMPL) {
+ /* FA-WWN is only for physical port */
+ if (qla_ini_mode_enabled(vha) &&
+ ha->flags.fawwpn_enabled &&
+ (rptid_entry->u.f1.flags &
+ VP_FLAGS_NAME_VALID)) {
+ memcpy(vha->port_name,
+ rptid_entry->u.f1.port_name,
+ WWN_SIZE);
+ }
- if (!MSB(stat)) {
- if (rptid_entry->vp_idx_map[1] & BIT_6)
- wwpn = rptid_entry->reserved_4 + 8;
+ vha->d_id.b.domain = rptid_entry->port_id[2];
+ vha->d_id.b.area = rptid_entry->port_id[1];
+ vha->d_id.b.al_pa = rptid_entry->port_id[0];
+ spin_lock_irqsave(&ha->vport_slock, flags);
+ qlt_update_vp_map(vha, SET_AL_PA);
+ spin_unlock_irqrestore(&ha->vport_slock, flags);
}
- memcpy(vha->port_name, wwpn, WWN_SIZE);
+
fc_host_port_name(vha->host) =
wwn_to_u64(vha->port_name);
- ql_dbg(ql_dbg_mbx, vha, 0x1018,
- "FA-WWN portname %016llx (%x)\n",
- fc_host_port_name(vha->host), MSB(stat));
- }
-
- vp = vha;
- if (vp_idx == 0)
- goto reg_needed;
- if (MSB(stat) != 0 && MSB(stat) != 2) {
- ql_dbg(ql_dbg_mbx, vha, 0x10ba,
- "Could not acquire ID for VP[%d].\n", vp_idx);
- return;
- }
+ if (qla_ini_mode_enabled(vha))
+ ql_dbg(ql_dbg_mbx, vha, 0x1018,
+ "FA-WWN portname %016llx (%x)\n",
+ fc_host_port_name(vha->host),
+ rptid_entry->vp_status);
- found = 0;
- spin_lock_irqsave(&ha->vport_slock, flags);
- list_for_each_entry(vp, &ha->vp_list, list) {
- if (vp_idx == vp->vp_idx) {
- found = 1;
- break;
+ set_bit(REGISTER_FC4_NEEDED, &vha->dpc_flags);
+ set_bit(REGISTER_FDMI_NEEDED, &vha->dpc_flags);
+ } else {
+ if (rptid_entry->vp_status != VP_STAT_COMPL &&
+ rptid_entry->vp_status != VP_STAT_ID_CHG) {
+ ql_dbg(ql_dbg_mbx, vha, 0x10ba,
+ "Could not acquire ID for VP[%d].\n",
+ rptid_entry->vp_idx);
+ return;
}
- }
- spin_unlock_irqrestore(&ha->vport_slock, flags);
- if (!found)
- return;
+ found = 0;
+ spin_lock_irqsave(&ha->vport_slock, flags);
+ list_for_each_entry(vp, &ha->vp_list, list) {
+ if (rptid_entry->vp_idx == vp->vp_idx) {
+ found = 1;
+ break;
+ }
+ }
+ spin_unlock_irqrestore(&ha->vport_slock, flags);
- vp->d_id.b.domain = rptid_entry->port_id[2];
- vp->d_id.b.area = rptid_entry->port_id[1];
- vp->d_id.b.al_pa = rptid_entry->port_id[0];
+ if (!found)
+ return;
- /*
- * Cannot configure here as we are still sitting on the
- * response queue. Handle it in dpc context.
- */
- set_bit(VP_IDX_ACQUIRED, &vp->vp_flags);
+ vp->d_id.b.domain = rptid_entry->port_id[2];
+ vp->d_id.b.area = rptid_entry->port_id[1];
+ vp->d_id.b.al_pa = rptid_entry->port_id[0];
+ spin_lock_irqsave(&ha->vport_slock, flags);
+ qlt_update_vp_map(vp, SET_AL_PA);
+ spin_unlock_irqrestore(&ha->vport_slock, flags);
-reg_needed:
- set_bit(REGISTER_FC4_NEEDED, &vp->dpc_flags);
- set_bit(REGISTER_FDMI_NEEDED, &vp->dpc_flags);
+ /*
+ * Cannot configure here as we are still sitting on the
+ * response queue. Handle it in dpc context.
+ */
+ set_bit(VP_IDX_ACQUIRED, &vp->vp_flags);
+ set_bit(REGISTER_FC4_NEEDED, &vp->dpc_flags);
+ set_bit(REGISTER_FDMI_NEEDED, &vp->dpc_flags);
+ }
set_bit(VP_DPC_NEEDED, &vha->dpc_flags);
qla2xxx_wake_dpc(vha);
+ } else if (rptid_entry->format == 2) {
+ ql_dbg(ql_dbg_async, vha, 0xffff,
+ "RIDA: format 2/N2N Primary port id %02x%02x%02x.\n",
+ rptid_entry->port_id[2], rptid_entry->port_id[1],
+ rptid_entry->port_id[0]);
+
+ ql_dbg(ql_dbg_async, vha, 0xffff,
+ "N2N: Remote WWPN %8phC.\n",
+ rptid_entry->u.f2.port_name);
+
+ /* N2N. direct connect */
+ vha->d_id.b.domain = rptid_entry->port_id[2];
+ vha->d_id.b.area = rptid_entry->port_id[1];
+ vha->d_id.b.al_pa = rptid_entry->port_id[0];
+
+ spin_lock_irqsave(&ha->vport_slock, flags);
+ qlt_update_vp_map(vha, SET_AL_PA);
+ spin_unlock_irqrestore(&ha->vport_slock, flags);
}
}
static void
qla2x00_fxdisc_iocb_timeout(void *data)
{
- srb_t *sp = (srb_t *)data;
+ srb_t *sp = data;
struct srb_iocb *lio = &sp->u.iocb_cmd;
complete(&lio->u.fxiocb.fxiocb_comp);
}
static void
-qla2x00_fxdisc_sp_done(void *data, void *ptr, int res)
+qla2x00_fxdisc_sp_done(void *ptr, int res)
{
- srb_t *sp = (srb_t *)ptr;
+ srb_t *sp = ptr;
struct srb_iocb *lio = &sp->u.iocb_cmd;
complete(&lio->u.fxiocb.fxiocb_comp);
dma_free_coherent(&ha->pdev->dev, fdisc->u.fxiocb.req_len,
fdisc->u.fxiocb.req_addr, fdisc->u.fxiocb.req_dma_handle);
done_free_sp:
- sp->free(vha, sp);
+ sp->free(sp);
done:
return rval;
}
qlafx00_handle_sense(srb_t *sp, uint8_t *sense_data, uint32_t par_sense_len,
uint32_t sense_len, struct rsp_que *rsp, int res)
{
- struct scsi_qla_host *vha = sp->fcport->vha;
+ struct scsi_qla_host *vha = sp->vha;
struct scsi_cmnd *cp = GET_CMD_SP(sp);
uint32_t track_sense_len;
if (sense_len) {
ql_dbg(ql_dbg_io + ql_dbg_buffer, vha, 0x3039,
"Check condition Sense data, nexus%ld:%d:%llu cmd=%p.\n",
- sp->fcport->vha->host_no, cp->device->id, cp->device->lun,
+ sp->vha->host_no, cp->device->id, cp->device->lun,
cp);
ql_dump_buffer(ql_dbg_io + ql_dbg_buffer, vha, 0x3049,
cp->sense_buffer, sense_len);
(sstatus & cpu_to_le16((uint16_t)SS_RESPONSE_INFO_LEN_VALID)))
cpstatus = cpu_to_le16((uint16_t)CS_INCOMPLETE);
tmf->u.tmf.comp_status = cpstatus;
- sp->done(vha, sp, 0);
+ sp->done(sp, 0);
}
static void
abt = &sp->u.iocb_cmd;
abt->u.abt.comp_status = pkt->tgt_id_sts;
- sp->done(vha, sp, 0);
+ sp->done(sp, 0);
}
static void
bsg_reply->reply_payload_rcv_len =
bsg_job->reply_payload.payload_len;
}
- sp->done(vha, sp, res);
+ sp->done(sp, res);
}
/**
par_sense_len, rsp_info_len);
if (rsp->status_srb == NULL)
- sp->done(ha, sp, res);
+ sp->done(sp, res);
}
/**
/* Place command on done queue. */
if (sense_len == 0) {
rsp->status_srb = NULL;
- sp->done(ha, sp, cp->result);
+ sp->done(sp, cp->result);
}
}
sp = qla2x00_get_sp_from_handle(vha, func, req, pkt);
if (sp) {
- sp->done(ha, sp, res);
+ sp->done(sp, res);
return;
}
cont_a64_entry_t lcont_pkt;
cont_a64_entry_t *cont_pkt;
- vha = sp->fcport->vha;
+ vha = sp->vha;
req = vha->req;
cmd = GET_CMD_SP(sp);
struct req_que *req = NULL;
struct rsp_que *rsp = NULL;
struct scsi_cmnd *cmd = GET_CMD_SP(sp);
- struct scsi_qla_host *vha = sp->fcport->vha;
+ struct scsi_qla_host *vha = sp->vha;
struct qla_hw_data *ha = vha->hw;
struct cmd_type_7_fx00 *cmd_pkt;
struct cmd_type_7_fx00 lcmd_pkt;
qlafx00_tm_iocb(srb_t *sp, struct tsk_mgmt_entry_fx00 *ptm_iocb)
{
struct srb_iocb *fxio = &sp->u.iocb_cmd;
- scsi_qla_host_t *vha = sp->fcport->vha;
+ scsi_qla_host_t *vha = sp->vha;
struct req_que *req = vha->req;
struct tsk_mgmt_entry_fx00 tm_iocb;
struct scsi_lun llun;
qlafx00_abort_iocb(srb_t *sp, struct abort_iocb_entry_fx00 *pabt_iocb)
{
struct srb_iocb *fxio = &sp->u.iocb_cmd;
- scsi_qla_host_t *vha = sp->fcport->vha;
+ scsi_qla_host_t *vha = sp->vha;
struct req_que *req = vha->req;
struct abort_iocb_entry_fx00 abt_iocb;
REQUEST_ENTRY_SIZE);
cont_pkt =
qlafx00_prep_cont_type1_iocb(
- sp->fcport->vha->req,
- &lcont_pkt);
+ sp->vha->req, &lcont_pkt);
cur_dsd = (__le32 *)
lcont_pkt.dseg_0_address;
avail_dsds = 5;
&lcont_pkt, REQUEST_ENTRY_SIZE);
ql_dump_buffer(
ql_dbg_user + ql_dbg_verbose,
- sp->fcport->vha, 0x3042,
+ sp->vha, 0x3042,
(uint8_t *)&lcont_pkt,
REQUEST_ENTRY_SIZE);
}
memcpy_toio((void __iomem *)cont_pkt,
&lcont_pkt, REQUEST_ENTRY_SIZE);
ql_dump_buffer(ql_dbg_user + ql_dbg_verbose,
- sp->fcport->vha, 0x3043,
+ sp->vha, 0x3043,
(uint8_t *)&lcont_pkt, REQUEST_ENTRY_SIZE);
}
}
REQUEST_ENTRY_SIZE);
cont_pkt =
qlafx00_prep_cont_type1_iocb(
- sp->fcport->vha->req,
- &lcont_pkt);
+ sp->vha->req, &lcont_pkt);
cur_dsd = (__le32 *)
lcont_pkt.dseg_0_address;
avail_dsds = 5;
REQUEST_ENTRY_SIZE);
ql_dump_buffer(
ql_dbg_user + ql_dbg_verbose,
- sp->fcport->vha, 0x3045,
+ sp->vha, 0x3045,
(uint8_t *)&lcont_pkt,
REQUEST_ENTRY_SIZE);
}
memcpy_toio((void __iomem *)cont_pkt,
&lcont_pkt, REQUEST_ENTRY_SIZE);
ql_dump_buffer(ql_dbg_user + ql_dbg_verbose,
- sp->fcport->vha, 0x3046,
+ sp->vha, 0x3046,
(uint8_t *)&lcont_pkt, REQUEST_ENTRY_SIZE);
}
}
}
ql_dump_buffer(ql_dbg_user + ql_dbg_verbose,
- sp->fcport->vha, 0x3047,
+ sp->vha, 0x3047,
(uint8_t *)&fx_iocb, sizeof(struct fxdisc_entry_fx00));
memcpy_toio((void __iomem *)pfxiocb, &fx_iocb,
"0 (Default) Do not set fw option. "
"1 - Set fw option to hold ABTS.");
+int ql2xmvasynctoatio = 1;
+module_param(ql2xmvasynctoatio, int, S_IRUGO|S_IWUSR);
+MODULE_PARM_DESC(ql2xmvasynctoatio,
+ "Move PUREX, ABTS RX and RIDA IOCBs to ATIOQ"
+ "0 (Default). Do not move IOCBs"
+ "1 - Move IOCBs.");
+
/*
* SCSI host template entry points
*/
}
void
-qla2x00_sp_free_dma(void *vha, void *ptr)
+qla2x00_sp_free_dma(void *ptr)
{
- srb_t *sp = (srb_t *)ptr;
+ srb_t *sp = ptr;
+ struct qla_hw_data *ha = sp->vha->hw;
struct scsi_cmnd *cmd = GET_CMD_SP(sp);
- struct qla_hw_data *ha = sp->fcport->vha->hw;
void *ctx = GET_CMD_CTX_SP(sp);
if (sp->flags & SRB_DMA_VALID) {
}
CMD_SP(cmd) = NULL;
- qla2x00_rel_sp(sp->fcport->vha, sp);
+ qla2x00_rel_sp(sp);
}
void
-qla2x00_sp_compl(void *data, void *ptr, int res)
+qla2x00_sp_compl(void *ptr, int res)
{
- struct qla_hw_data *ha = (struct qla_hw_data *)data;
- srb_t *sp = (srb_t *)ptr;
+ srb_t *sp = ptr;
struct scsi_cmnd *cmd = GET_CMD_SP(sp);
cmd->result = res;
if (atomic_read(&sp->ref_count) == 0) {
- ql_dbg(ql_dbg_io, sp->fcport->vha, 0x3015,
+ ql_dbg(ql_dbg_io, sp->vha, 0x3015,
"SP reference-count to ZERO -- sp=%p cmd=%p.\n",
sp, GET_CMD_SP(sp));
if (ql2xextended_error_logging & ql_dbg_io)
if (!atomic_dec_and_test(&sp->ref_count))
return;
- qla2x00_sp_free_dma(ha, sp);
+ qla2x00_sp_free_dma(sp);
cmd->scsi_done(cmd);
}
void
-qla2xxx_qpair_sp_free_dma(void *vha, void *ptr)
+qla2xxx_qpair_sp_free_dma(void *ptr)
{
srb_t *sp = (srb_t *)ptr;
struct scsi_cmnd *cmd = GET_CMD_SP(sp);
}
void
-qla2xxx_qpair_sp_compl(void *data, void *ptr, int res)
+qla2xxx_qpair_sp_compl(void *ptr, int res)
{
- srb_t *sp = (srb_t *)ptr;
+ srb_t *sp = ptr;
struct scsi_cmnd *cmd = GET_CMD_SP(sp);
cmd->result = res;
if (!atomic_dec_and_test(&sp->ref_count))
return;
- qla2xxx_qpair_sp_free_dma(sp->fcport->vha, sp);
+ qla2xxx_qpair_sp_free_dma(sp);
cmd->scsi_done(cmd);
}
return 0;
qc24_host_busy_free_sp:
- qla2x00_sp_free_dma(ha, sp);
+ qla2x00_sp_free_dma(sp);
qc24_host_busy:
return SCSI_MLQUEUE_HOST_BUSY;
return 0;
qc24_host_busy_free_sp:
- qla2xxx_qpair_sp_free_dma(vha, sp);
+ qla2xxx_qpair_sp_free_dma(sp);
qc24_host_busy:
return SCSI_MLQUEUE_HOST_BUSY;
return (return_status);
}
+static inline int test_fcport_count(scsi_qla_host_t *vha)
+{
+ struct qla_hw_data *ha = vha->hw;
+ unsigned long flags;
+ int res;
+
+ spin_lock_irqsave(&ha->tgt.sess_lock, flags);
+ ql_dbg(ql_dbg_init, vha, 0xffff,
+ "tgt %p, fcport_count=%d\n",
+ vha, vha->fcport_count);
+ res = (vha->fcport_count == 0);
+ spin_unlock_irqrestore(&ha->tgt.sess_lock, flags);
+
+ return res;
+}
+
+/*
+ * qla2x00_wait_for_sess_deletion can only be called from remove_one.
+ * it has dependency on UNLOADING flag to stop device discovery
+ */
+static void
+qla2x00_wait_for_sess_deletion(scsi_qla_host_t *vha)
+{
+ qla2x00_mark_all_devices_lost(vha, 0);
+
+ wait_event(vha->fcport_waitQ, test_fcport_count(vha));
+}
+
/*
* qla2x00_wait_for_hba_ready
* Wait till the HBA is ready before doing driver unload
}
spin_lock_irqsave(&ha->hardware_lock, flags);
- sp->done(ha, sp, 0);
+ sp->done(sp, 0);
spin_unlock_irqrestore(&ha->hardware_lock, flags);
/* Did the command return during mailbox execution? */
continue;
if (sp->type != SRB_SCSI_CMD)
continue;
- if (vha->vp_idx != sp->fcport->vha->vp_idx)
+ if (vha->vp_idx != sp->vha->vp_idx)
continue;
match = 0;
cmd = GET_CMD_SP(sp);
spin_lock_irqsave(&ha->hardware_lock, flags);
}
req->outstanding_cmds[cnt] = NULL;
- sp->done(vha, sp, res);
+ sp->done(sp, res);
}
}
}
/* Determine queue resources */
ha->max_req_queues = ha->max_rsp_queues = 1;
+ ha->msix_count = QLA_BASE_VECTORS;
if (!ql2xmqsupport || (!IS_QLA25XX(ha) && !IS_QLA81XX(ha)))
goto mqiobase_exit;
"BAR 3 not enabled.\n");
mqiobase_exit:
- ha->msix_count = ha->max_rsp_queues + 1;
ql_dbg_pci(ql_dbg_init, ha->pdev, 0x001c,
- "MSIX Count:%d.\n", ha->msix_count);
+ "MSIX Count: %d.\n", ha->msix_count);
return (0);
iospace_error_exit:
/* 83XX 26XX always use MQ type access for queues
* - mbar 2, a.k.a region 4 */
ha->max_req_queues = ha->max_rsp_queues = 1;
+ ha->msix_count = QLA_BASE_VECTORS;
ha->mqiobase = ioremap(pci_resource_start(ha->pdev, 4),
pci_resource_len(ha->pdev, 4));
if (ql2xmqsupport) {
/* MB interrupt uses 1 vector */
ha->max_req_queues = ha->msix_count - 1;
- ha->max_rsp_queues = ha->max_req_queues;
/* ATIOQ needs 1 vector. That's 1 less QPair */
if (QLA_TGT_MODE_ENABLED())
ha->max_req_queues--;
+ ha->max_rsp_queues = ha->max_req_queues;
+
/* Queue pairs is the max value minus
* the base queue pair */
ha->max_qpairs = ha->max_req_queues - 1;
"BAR 1 not enabled.\n");
mqiobase_exit:
- ha->msix_count = ha->max_rsp_queues + 1;
- if (QLA_TGT_MODE_ENABLED())
- ha->msix_count++;
-
- qlt_83xx_iospace_config(ha);
-
ql_dbg_pci(ql_dbg_init, ha->pdev, 0x011f,
- "MSIX Count:%d.\n", ha->msix_count);
+ "MSIX Count: %d.\n", ha->msix_count);
return 0;
iospace_error_exit:
ql_dbg(ql_dbg_init, base_vha, 0x00f2,
"Init done and hba is online.\n");
- if (qla_ini_mode_enabled(base_vha))
+ if (qla_ini_mode_enabled(base_vha) ||
+ qla_dual_mode_enabled(base_vha))
scsi_scan_host(host);
else
ql_dbg(ql_dbg_init, base_vha, 0x0122,
* resources.
*/
if (!atomic_read(&pdev->enable_cnt)) {
+ dma_free_coherent(&ha->pdev->dev, base_vha->gnl.size,
+ base_vha->gnl.l, base_vha->gnl.ldma);
+
scsi_host_put(base_vha->host);
kfree(ha);
pci_set_drvdata(pdev, NULL);
return;
}
-
qla2x00_wait_for_hba_ready(base_vha);
- /* if UNLOAD flag is already set, then continue unload,
+ /*
+ * if UNLOAD flag is already set, then continue unload,
* where it was set first.
*/
if (test_bit(UNLOADING, &base_vha->dpc_flags))
return;
set_bit(UNLOADING, &base_vha->dpc_flags);
+ dma_free_coherent(&ha->pdev->dev,
+ base_vha->gnl.size, base_vha->gnl.l, base_vha->gnl.ldma);
if (IS_QLAFX00(ha))
qlafx00_driver_shutdown(base_vha, 20);
qla2xxx_wake_dpc(base_vha);
} else {
int now;
- if (rport)
+ if (rport) {
+ ql_dbg(ql_dbg_disc, fcport->vha, 0xffff,
+ "%s %8phN. rport %p roles %x \n",
+ __func__, fcport->port_name, rport,
+ rport->roles);
fc_remote_port_delete(rport);
+ }
qlt_do_generation_tick(vha, &now);
- qlt_fc_port_deleted(vha, fcport, now);
}
}
fcport->login_retry = vha->hw->login_retry_count;
ql_dbg(ql_dbg_disc, vha, 0x2067,
- "Port login retry %8phN, id = 0x%04x retry cnt=%d.\n",
+ "Port login retry %8phN, lid 0x%04x retry cnt=%d.\n",
fcport->port_name, fcport->loop_id, fcport->login_retry);
}
}
{
fc_port_t *fcport;
+ ql_dbg(ql_dbg_disc, vha, 0xffff,
+ "Mark all dev lost\n");
+
list_for_each_entry(fcport, &vha->vp_fcports, list) {
+ fcport->scan_state = 0;
+ qlt_schedule_sess_for_deletion_lock(fcport);
+
if (vha->vp_idx != 0 && vha->vp_idx != fcport->vha->vp_idx)
continue;
struct scsi_qla_host *vha = NULL;
host = scsi_host_alloc(sht, sizeof(scsi_qla_host_t));
- if (host == NULL) {
+ if (!host) {
ql_log_pci(ql_log_fatal, ha->pdev, 0x0107,
"Failed to allocate host from the scsi layer, aborting.\n");
- goto fail;
+ return NULL;
}
/* Clear our data area */
INIT_LIST_HEAD(&vha->logo_list);
INIT_LIST_HEAD(&vha->plogi_ack_list);
INIT_LIST_HEAD(&vha->qp_list);
+ INIT_LIST_HEAD(&vha->gnl.fcports);
spin_lock_init(&vha->work_lock);
spin_lock_init(&vha->cmd_list_lock);
+ init_waitqueue_head(&vha->fcport_waitQ);
+
+ vha->gnl.size = sizeof(struct get_name_list_extended) *
+ (ha->max_loop_id + 1);
+ vha->gnl.l = dma_alloc_coherent(&ha->pdev->dev,
+ vha->gnl.size, &vha->gnl.ldma, GFP_KERNEL);
+ if (!vha->gnl.l) {
+ ql_log(ql_log_fatal, vha, 0xffff,
+ "Alloc failed for name list.\n");
+ scsi_remove_host(vha->host);
+ return NULL;
+ }
sprintf(vha->host_str, "%s_%ld", QLA2XXX_DRIVER_NAME, vha->host_no);
ql_dbg(ql_dbg_init, vha, 0x0041,
dev_name(&(ha->pdev->dev)));
return vha;
-
-fail:
- return vha;
}
-static struct qla_work_evt *
+struct qla_work_evt *
qla2x00_alloc_work(struct scsi_qla_host *vha, enum qla_work_type type)
{
struct qla_work_evt *e;
return e;
}
-static int
+int
qla2x00_post_work(struct scsi_qla_host *vha, struct qla_work_evt *e)
{
unsigned long flags;
}
qla2x00_post_async_work(login, QLA_EVT_ASYNC_LOGIN);
-qla2x00_post_async_work(login_done, QLA_EVT_ASYNC_LOGIN_DONE);
qla2x00_post_async_work(logout, QLA_EVT_ASYNC_LOGOUT);
qla2x00_post_async_work(logout_done, QLA_EVT_ASYNC_LOGOUT_DONE);
qla2x00_post_async_work(adisc, QLA_EVT_ASYNC_ADISC);
return qla2x00_post_work(vha, e);
}
+int qla24xx_post_upd_fcport_work(struct scsi_qla_host *vha, fc_port_t *fcport)
+{
+ struct qla_work_evt *e;
+
+ e = qla2x00_alloc_work(vha, QLA_EVT_UPD_FCPORT);
+ if (!e)
+ return QLA_FUNCTION_FAILED;
+
+ e->u.fcport.fcport = fcport;
+ return qla2x00_post_work(vha, e);
+}
+
+static
+void qla24xx_create_new_sess(struct scsi_qla_host *vha, struct qla_work_evt *e)
+{
+ unsigned long flags;
+ fc_port_t *fcport = NULL;
+ struct qlt_plogi_ack_t *pla =
+ (struct qlt_plogi_ack_t *)e->u.new_sess.pla;
+
+ spin_lock_irqsave(&vha->hw->tgt.sess_lock, flags);
+ fcport = qla2x00_find_fcport_by_wwpn(vha, e->u.new_sess.port_name, 1);
+ if (fcport) {
+ fcport->d_id = e->u.new_sess.id;
+ if (pla) {
+ fcport->fw_login_state = DSC_LS_PLOGI_PEND;
+ qlt_plogi_ack_link(vha, pla, fcport, QLT_PLOGI_LINK_SAME_WWN);
+ /* we took an extra ref_count to prevent PLOGI ACK when
+ * fcport/sess has not been created.
+ */
+ pla->ref_count--;
+ }
+ } else {
+ fcport = qla2x00_alloc_fcport(vha, GFP_KERNEL);
+ if (fcport) {
+ fcport->d_id = e->u.new_sess.id;
+ fcport->scan_state = QLA_FCPORT_FOUND;
+ fcport->flags |= FCF_FABRIC_DEVICE;
+ fcport->fw_login_state = DSC_LS_PLOGI_PEND;
+
+ memcpy(fcport->port_name, e->u.new_sess.port_name,
+ WWN_SIZE);
+ list_add_tail(&fcport->list, &vha->vp_fcports);
+
+ if (pla) {
+ qlt_plogi_ack_link(vha, pla, fcport,
+ QLT_PLOGI_LINK_SAME_WWN);
+ pla->ref_count--;
+ }
+ }
+ }
+ spin_unlock_irqrestore(&vha->hw->tgt.sess_lock, flags);
+
+ if (fcport) {
+ if (pla)
+ qlt_plogi_ack_unref(vha, pla);
+ else
+ qla24xx_async_gnl(vha, fcport);
+ }
+}
+
void
qla2x00_do_work(struct scsi_qla_host *vha)
{
qla2x00_async_login(vha, e->u.logio.fcport,
e->u.logio.data);
break;
- case QLA_EVT_ASYNC_LOGIN_DONE:
- qla2x00_async_login_done(vha, e->u.logio.fcport,
- e->u.logio.data);
- break;
case QLA_EVT_ASYNC_LOGOUT:
qla2x00_async_logout(vha, e->u.logio.fcport);
break;
case QLA_EVT_AENFX:
qlafx00_process_aen(vha, e);
break;
+ case QLA_EVT_GIDPN:
+ qla24xx_async_gidpn(vha, e->u.fcport.fcport);
+ break;
+ case QLA_EVT_GPNID:
+ qla24xx_async_gpnid(vha, &e->u.gpnid.id);
+ break;
+ case QLA_EVT_GPNID_DONE:
+ qla24xx_async_gpnid_done(vha, e->u.iosb.sp);
+ break;
+ case QLA_EVT_NEW_SESS:
+ qla24xx_create_new_sess(vha, e);
+ break;
+ case QLA_EVT_GPDB:
+ qla24xx_async_gpdb(vha, e->u.fcport.fcport,
+ e->u.fcport.opt);
+ break;
+ case QLA_EVT_GPSC:
+ qla24xx_async_gpsc(vha, e->u.fcport.fcport);
+ break;
+ case QLA_EVT_UPD_FCPORT:
+ qla2x00_update_fcport(vha, e->u.fcport.fcport);
+ break;
+ case QLA_EVT_GNL:
+ qla24xx_async_gnl(vha, e->u.fcport.fcport);
+ break;
+ case QLA_EVT_NACK:
+ qla24xx_do_nack_work(vha, e);
+ break;
}
if (e->flags & QLA_EVT_FLAG_FREE)
kfree(e);
{
fc_port_t *fcport;
int status;
- uint16_t next_loopid = 0;
- struct qla_hw_data *ha = vha->hw;
- uint16_t data[2];
+ struct event_arg ea;
list_for_each_entry(fcport, &vha->vp_fcports, list) {
/*
fcport->login_retry && !(fcport->flags & FCF_ASYNC_SENT)) {
fcport->login_retry--;
if (fcport->flags & FCF_FABRIC_DEVICE) {
- if (fcport->flags & FCF_FCP2_DEVICE)
- ha->isp_ops->fabric_logout(vha,
- fcport->loop_id,
- fcport->d_id.b.domain,
- fcport->d_id.b.area,
- fcport->d_id.b.al_pa);
-
- if (fcport->loop_id == FC_NO_LOOP_ID) {
- fcport->loop_id = next_loopid =
- ha->min_external_loopid;
- status = qla2x00_find_new_loop_id(
- vha, fcport);
- if (status != QLA_SUCCESS) {
- /* Ran out of IDs to use */
- break;
- }
- }
-
- if (IS_ALOGIO_CAPABLE(ha)) {
- fcport->flags |= FCF_ASYNC_SENT;
- data[0] = 0;
- data[1] = QLA_LOGIO_LOGIN_RETRIED;
- status = qla2x00_post_async_login_work(
- vha, fcport, data);
- if (status == QLA_SUCCESS)
- continue;
- /* Attempt a retry. */
- status = 1;
- } else {
- status = qla2x00_fabric_login(vha,
- fcport, &next_loopid);
- if (status == QLA_SUCCESS) {
- int status2;
- uint8_t opts;
-
- opts = 0;
- if (fcport->flags &
- FCF_FCP2_DEVICE)
- opts |= BIT_1;
- status2 =
- qla2x00_get_port_database(
- vha, fcport, opts);
- if (status2 != QLA_SUCCESS)
- status = 1;
- }
- }
- } else
+ ql_dbg(ql_dbg_disc, fcport->vha, 0xffff,
+ "%s %8phC DS %d LS %d\n", __func__,
+ fcport->port_name, fcport->disc_state,
+ fcport->fw_login_state);
+ memset(&ea, 0, sizeof(ea));
+ ea.event = FCME_RELOGIN;
+ ea.fcport = fcport;
+ qla2x00_fcport_event_handler(vha, &ea);
+ } else {
status = qla2x00_local_device_login(vha,
fcport);
+ if (status == QLA_SUCCESS) {
+ fcport->old_loop_id = fcport->loop_id;
+ ql_dbg(ql_dbg_disc, vha, 0x2003,
+ "Port login OK: logged in ID 0x%x.\n",
+ fcport->loop_id);
+ qla2x00_update_fcport(vha, fcport);
+ } else if (status == 1) {
+ set_bit(RELOGIN_NEEDED, &vha->dpc_flags);
+ /* retry the login again */
+ ql_dbg(ql_dbg_disc, vha, 0x2007,
+ "Retrying %d login again loop_id 0x%x.\n",
+ fcport->login_retry,
+ fcport->loop_id);
+ } else {
+ fcport->login_retry = 0;
+ }
- if (status == QLA_SUCCESS) {
- fcport->old_loop_id = fcport->loop_id;
-
- ql_dbg(ql_dbg_disc, vha, 0x2003,
- "Port login OK: logged in ID 0x%x.\n",
- fcport->loop_id);
-
- qla2x00_update_fcport(vha, fcport);
-
- } else if (status == 1) {
- set_bit(RELOGIN_NEEDED, &vha->dpc_flags);
- /* retry the login again */
- ql_dbg(ql_dbg_disc, vha, 0x2007,
- "Retrying %d login again loop_id 0x%x.\n",
- fcport->login_retry, fcport->loop_id);
- } else {
- fcport->login_retry = 0;
+ if (fcport->login_retry == 0 &&
+ status != QLA_SUCCESS)
+ qla2x00_clear_loop_id(fcport);
}
-
- if (fcport->login_retry == 0 && status != QLA_SUCCESS)
- qla2x00_clear_loop_id(fcport);
}
if (test_bit(LOOP_RESYNC_NEEDED, &vha->dpc_flags))
break;
struct pci_dev *pdev = ha->pdev;
scsi_qla_host_t *base_vha = pci_get_drvdata(ha->pdev);
- /* if UNLOAD flag is already set, then continue unload,
+ /*
+ * if UNLOAD flag is already set, then continue unload,
* where it was set first.
*/
if (test_bit(UNLOADING, &base_vha->dpc_flags))
ql_log(ql_log_warn, base_vha, 0x015b,
"Disabling adapter.\n");
+ qla2x00_wait_for_sess_deletion(base_vha);
+
set_bit(UNLOADING, &base_vha->dpc_flags);
qla2x00_delete_all_vps(ha, base_vha);
qla2x00_update_fcports(base_vha);
}
- if (test_bit(SCR_PENDING, &base_vha->dpc_flags)) {
- int ret;
- ret = qla2x00_send_change_request(base_vha, 0x3, 0);
- if (ret != QLA_SUCCESS)
- ql_log(ql_log_warn, base_vha, 0x121,
- "Failed to enable receiving of RSCN "
- "requests: 0x%x.\n", ret);
- clear_bit(SCR_PENDING, &base_vha->dpc_flags);
- }
-
if (IS_QLAFX00(ha))
goto loop_resync_check;
"disabled on enabling target mode and then on disabling target mode "
"enabled back; "
"\"disabled\" - initiator mode will never be enabled; "
+ "\"dual\" - Initiator Modes will be enabled. Target Mode can be activated "
+ "when ready "
"\"enabled\" (default) - initiator mode will always stay enabled.");
+static int ql_dm_tgt_ex_pct = 50;
+module_param(ql_dm_tgt_ex_pct, int, S_IRUGO|S_IWUSR);
+MODULE_PARM_DESC(ql_dm_tgt_ex_pct,
+ "For Dual Mode (qlini_mode=dual), this parameter determines "
+ "the percentage of exchanges/cmds FW will allocate resources "
+ "for Target mode.");
+
int ql2x_ini_mode = QLA2XXX_INI_MODE_EXCLUSIVE;
static int temp_sam_status = SAM_STAT_BUSY;
static void qlt_24xx_atio_pkt(struct scsi_qla_host *ha,
struct atio_from_isp *pkt, uint8_t);
static void qlt_response_pkt(struct scsi_qla_host *ha, response_t *pkt);
-static int qlt_issue_task_mgmt(struct qla_tgt_sess *sess, uint32_t lun,
+static int qlt_issue_task_mgmt(struct fc_port *sess, u64 lun,
int fn, void *iocb, int flags);
static void qlt_send_term_exchange(struct scsi_qla_host *ha, struct qla_tgt_cmd
*cmd, struct atio_from_isp *atio, int ha_locked, int ul_abort);
-static void qlt_reject_free_srr_imm(struct scsi_qla_host *ha,
- struct qla_tgt_srr_imm *imm, int ha_lock);
static void qlt_abort_cmd_on_host_reset(struct scsi_qla_host *vha,
struct qla_tgt_cmd *cmd);
static void qlt_alloc_qfull_cmd(struct scsi_qla_host *vha,
uint16_t srr_flags, uint16_t srr_reject_code, uint8_t srr_explan);
static void qlt_send_term_imm_notif(struct scsi_qla_host *vha,
struct imm_ntfy_from_isp *imm, int ha_locked);
+static struct fc_port *qlt_create_sess(struct scsi_qla_host *vha,
+ fc_port_t *fcport, bool local);
+void qlt_unreg_sess(struct fc_port *sess);
/*
* Global Variables
*/
wmb();
}
-/* ha->hardware_lock supposed to be held on entry (to protect tgt->sess_list) */
-static struct qla_tgt_sess *qlt_find_sess_by_port_name(
- struct qla_tgt *tgt,
- const uint8_t *port_name)
-{
- struct qla_tgt_sess *sess;
-
- list_for_each_entry(sess, &tgt->sess_list, sess_list_entry) {
- if (!memcmp(sess->port_name, port_name, WWN_SIZE))
- return sess;
- }
-
- return NULL;
-}
-
/* Might release hw lock, then reaquire!! */
static inline int qlt_issue_marker(struct scsi_qla_host *vha, int vha_locked)
{
spin_unlock_irqrestore(&vha->hw->tgt.q_full_lock, flags);
}
+
+static void qlt_queue_unknown_atio(scsi_qla_host_t *vha,
+ struct atio_from_isp *atio, uint8_t ha_locked)
+{
+ struct qla_tgt_sess_op *u;
+ struct qla_tgt *tgt = vha->vha_tgt.qla_tgt;
+ unsigned long flags;
+
+ if (tgt->tgt_stop) {
+ ql_dbg(ql_dbg_async, vha, 0xffff,
+ "qla_target(%d): dropping unknown ATIO_TYPE7, "
+ "because tgt is being stopped", vha->vp_idx);
+ goto out_term;
+ }
+
+ u = kzalloc(sizeof(*u), GFP_ATOMIC);
+ if (u == NULL) {
+ ql_dbg(ql_dbg_async, vha, 0xffff,
+ "Alloc of struct unknown_atio (size %zd) failed", sizeof(*u));
+ /* It should be harmless and on the next retry should work well */
+ goto out_term;
+ }
+
+ u->vha = vha;
+ memcpy(&u->atio, atio, sizeof(*atio));
+ INIT_LIST_HEAD(&u->cmd_list);
+
+ spin_lock_irqsave(&vha->cmd_list_lock, flags);
+ list_add_tail(&u->cmd_list, &vha->unknown_atio_list);
+ spin_unlock_irqrestore(&vha->cmd_list_lock, flags);
+
+ schedule_delayed_work(&vha->unknown_atio_work, 1);
+
+out:
+ return;
+
+out_term:
+ qlt_send_term_exchange(vha, NULL, atio, ha_locked, 0);
+ goto out;
+}
+
+static void qlt_try_to_dequeue_unknown_atios(struct scsi_qla_host *vha,
+ uint8_t ha_locked)
+{
+ struct qla_tgt_sess_op *u, *t;
+ scsi_qla_host_t *host;
+ struct qla_tgt *tgt = vha->vha_tgt.qla_tgt;
+ unsigned long flags;
+ uint8_t queued = 0;
+
+ list_for_each_entry_safe(u, t, &vha->unknown_atio_list, cmd_list) {
+ if (u->aborted) {
+ ql_dbg(ql_dbg_async, vha, 0xffff,
+ "Freeing unknown %s %p, because of Abort",
+ "ATIO_TYPE7", u);
+ qlt_send_term_exchange(vha, NULL, &u->atio,
+ ha_locked, 0);
+ goto abort;
+ }
+
+ host = qlt_find_host_by_d_id(vha, u->atio.u.isp24.fcp_hdr.d_id);
+ if (host != NULL) {
+ ql_dbg(ql_dbg_async, vha, 0xffff,
+ "Requeuing unknown ATIO_TYPE7 %p", u);
+ qlt_24xx_atio_pkt(host, &u->atio, ha_locked);
+ } else if (tgt->tgt_stop) {
+ ql_dbg(ql_dbg_async, vha, 0xffff,
+ "Freeing unknown %s %p, because tgt is being stopped",
+ "ATIO_TYPE7", u);
+ qlt_send_term_exchange(vha, NULL, &u->atio,
+ ha_locked, 0);
+ } else {
+ ql_dbg(ql_dbg_async, vha, 0xffff,
+ "u %p, vha %p, host %p, sched again..", u,
+ vha, host);
+ if (!queued) {
+ queued = 1;
+ schedule_delayed_work(&vha->unknown_atio_work,
+ 1);
+ }
+ continue;
+ }
+
+abort:
+ spin_lock_irqsave(&vha->cmd_list_lock, flags);
+ list_del(&u->cmd_list);
+ spin_unlock_irqrestore(&vha->cmd_list_lock, flags);
+ kfree(u);
+ }
+}
+
+void qlt_unknown_atio_work_fn(struct work_struct *work)
+{
+ struct scsi_qla_host *vha = container_of(to_delayed_work(work),
+ struct scsi_qla_host, unknown_atio_work);
+
+ qlt_try_to_dequeue_unknown_atios(vha, 0);
+}
+
static bool qlt_24xx_atio_pkt_all_vps(struct scsi_qla_host *vha,
struct atio_from_isp *atio, uint8_t ha_locked)
{
atio->u.isp24.fcp_hdr.d_id[0],
atio->u.isp24.fcp_hdr.d_id[1],
atio->u.isp24.fcp_hdr.d_id[2]);
+
+
+ qlt_queue_unknown_atio(vha, atio, ha_locked);
break;
}
+ if (unlikely(!list_empty(&vha->unknown_atio_list)))
+ qlt_try_to_dequeue_unknown_atios(vha, ha_locked);
+
qlt_24xx_atio_pkt(host, atio, ha_locked);
break;
}
break;
}
+ case VP_RPT_ID_IOCB_TYPE:
+ qla24xx_report_id_acquisition(vha,
+ (struct vp_rpt_id_entry_24xx *)atio);
+ break;
+
+ case ABTS_RECV_24XX:
+ {
+ struct abts_recv_from_24xx *entry =
+ (struct abts_recv_from_24xx *)atio;
+ struct scsi_qla_host *host = qlt_find_host_by_vp_idx(vha,
+ entry->vp_index);
+ if (unlikely(!host)) {
+ ql_dbg(ql_dbg_tgt, vha, 0xffff,
+ "qla_target(%d): Response pkt (ABTS_RECV_24XX) "
+ "received, with unknown vp_index %d\n",
+ vha->vp_idx, entry->vp_index);
+ break;
+ }
+ qlt_response_pkt(host, (response_t *)atio);
+ break;
+
+ }
+
+ /* case PUREX_IOCB_TYPE: ql2xmvasynctoatio */
+
default:
ql_dbg(ql_dbg_tgt, vha, 0xe040,
"qla_target(%d): Received unknown ATIO atio "
/*
* All qlt_plogi_ack_t operations are protected by hardware_lock
*/
+static int qla24xx_post_nack_work(struct scsi_qla_host *vha, fc_port_t *fcport,
+ struct imm_ntfy_from_isp *ntfy, int type)
+{
+ struct qla_work_evt *e;
+ e = qla2x00_alloc_work(vha, QLA_EVT_NACK);
+ if (!e)
+ return QLA_FUNCTION_FAILED;
+
+ e->u.nack.fcport = fcport;
+ e->u.nack.type = type;
+ memcpy(e->u.nack.iocb, ntfy, sizeof(struct imm_ntfy_from_isp));
+ return qla2x00_post_work(vha, e);
+}
+
+static
+void qla2x00_async_nack_sp_done(void *s, int res)
+{
+ struct srb *sp = (struct srb *)s;
+ struct scsi_qla_host *vha = sp->vha;
+ unsigned long flags;
+
+ ql_dbg(ql_dbg_disc, vha, 0xffff,
+ "Async done-%s res %x %8phC type %d\n",
+ sp->name, res, sp->fcport->port_name, sp->type);
+
+ spin_lock_irqsave(&vha->hw->tgt.sess_lock, flags);
+ sp->fcport->flags &= ~FCF_ASYNC_SENT;
+ sp->fcport->chip_reset = vha->hw->chip_reset;
+
+ switch (sp->type) {
+ case SRB_NACK_PLOGI:
+ sp->fcport->login_gen++;
+ sp->fcport->fw_login_state = DSC_LS_PLOGI_COMP;
+ sp->fcport->logout_on_delete = 1;
+ break;
+
+ case SRB_NACK_PRLI:
+ sp->fcport->fw_login_state = DSC_LS_PRLI_COMP;
+ sp->fcport->deleted = 0;
+
+ if (!sp->fcport->login_succ &&
+ !IS_SW_RESV_ADDR(sp->fcport->d_id)) {
+ sp->fcport->login_succ = 1;
+
+ vha->fcport_count++;
+
+ if (!IS_IIDMA_CAPABLE(vha->hw) ||
+ !vha->hw->flags.gpsc_supported) {
+ ql_dbg(ql_dbg_disc, vha, 0xffff,
+ "%s %d %8phC post upd_fcport fcp_cnt %d\n",
+ __func__, __LINE__,
+ sp->fcport->port_name,
+ vha->fcport_count);
+
+ qla24xx_post_upd_fcport_work(vha, sp->fcport);
+ } else {
+ ql_dbg(ql_dbg_disc, vha, 0xffff,
+ "%s %d %8phC post gpsc fcp_cnt %d\n",
+ __func__, __LINE__,
+ sp->fcport->port_name,
+ vha->fcport_count);
+
+ qla24xx_post_gpsc_work(vha, sp->fcport);
+ }
+ }
+ break;
+
+ case SRB_NACK_LOGO:
+ sp->fcport->login_gen++;
+ sp->fcport->fw_login_state = DSC_LS_PORT_UNAVAIL;
+ qlt_logo_completion_handler(sp->fcport, MBS_COMMAND_COMPLETE);
+ break;
+ }
+ spin_unlock_irqrestore(&vha->hw->tgt.sess_lock, flags);
+
+ sp->free(sp);
+}
+
+int qla24xx_async_notify_ack(scsi_qla_host_t *vha, fc_port_t *fcport,
+ struct imm_ntfy_from_isp *ntfy, int type)
+{
+ int rval = QLA_FUNCTION_FAILED;
+ srb_t *sp;
+ char *c = NULL;
+
+ fcport->flags |= FCF_ASYNC_SENT;
+ switch (type) {
+ case SRB_NACK_PLOGI:
+ fcport->fw_login_state = DSC_LS_PLOGI_PEND;
+ c = "PLOGI";
+ break;
+ case SRB_NACK_PRLI:
+ fcport->fw_login_state = DSC_LS_PRLI_PEND;
+ c = "PRLI";
+ break;
+ case SRB_NACK_LOGO:
+ fcport->fw_login_state = DSC_LS_LOGO_PEND;
+ c = "LOGO";
+ break;
+ }
+
+ sp = qla2x00_get_sp(vha, fcport, GFP_ATOMIC);
+ if (!sp)
+ goto done;
+
+ sp->type = type;
+ sp->name = "nack";
+
+ qla2x00_init_timer(sp, qla2x00_get_async_timeout(vha)+2);
+
+ sp->u.iocb_cmd.u.nack.ntfy = ntfy;
+
+ sp->done = qla2x00_async_nack_sp_done;
+
+ rval = qla2x00_start_sp(sp);
+ if (rval != QLA_SUCCESS)
+ goto done_free_sp;
+
+ ql_dbg(ql_dbg_disc, vha, 0xffff,
+ "Async-%s %8phC hndl %x %s\n",
+ sp->name, fcport->port_name, sp->handle, c);
+
+ return rval;
+
+done_free_sp:
+ sp->free(sp);
+done:
+ fcport->flags &= ~FCF_ASYNC_SENT;
+ return rval;
+}
+
+void qla24xx_do_nack_work(struct scsi_qla_host *vha, struct qla_work_evt *e)
+{
+ fc_port_t *t;
+ unsigned long flags;
+
+ switch (e->u.nack.type) {
+ case SRB_NACK_PRLI:
+ mutex_lock(&vha->vha_tgt.tgt_mutex);
+ t = qlt_create_sess(vha, e->u.nack.fcport, 0);
+ mutex_unlock(&vha->vha_tgt.tgt_mutex);
+ if (t) {
+ ql_log(ql_log_info, vha, 0xffff,
+ "%s create sess success %p", __func__, t);
+ spin_lock_irqsave(&vha->hw->tgt.sess_lock, flags);
+ /* create sess has an extra kref */
+ vha->hw->tgt.tgt_ops->put_sess(e->u.nack.fcport);
+ spin_unlock_irqrestore(&vha->hw->tgt.sess_lock, flags);
+ }
+ break;
+ }
+ qla24xx_async_notify_ack(vha, e->u.nack.fcport,
+ (struct imm_ntfy_from_isp*)e->u.nack.iocb, e->u.nack.type);
+}
+
+void qla24xx_delete_sess_fn(struct work_struct *work)
+{
+ fc_port_t *fcport = container_of(work, struct fc_port, del_work);
+ struct qla_hw_data *ha = fcport->vha->hw;
+ unsigned long flags;
+
+ spin_lock_irqsave(&ha->tgt.sess_lock, flags);
+
+ if (fcport->se_sess) {
+ ha->tgt.tgt_ops->shutdown_sess(fcport);
+ ha->tgt.tgt_ops->put_sess(fcport);
+ } else {
+ qlt_unreg_sess(fcport);
+ }
+ spin_unlock_irqrestore(&ha->tgt.sess_lock, flags);
+}
+
+/*
+ * Called from qla2x00_reg_remote_port()
+ */
+void qlt_fc_port_added(struct scsi_qla_host *vha, fc_port_t *fcport)
+{
+ struct qla_hw_data *ha = vha->hw;
+ struct qla_tgt *tgt = vha->vha_tgt.qla_tgt;
+ struct fc_port *sess = fcport;
+ unsigned long flags;
+
+ if (!vha->hw->tgt.tgt_ops)
+ return;
+
+ spin_lock_irqsave(&ha->tgt.sess_lock, flags);
+ if (tgt->tgt_stop) {
+ spin_unlock_irqrestore(&ha->tgt.sess_lock, flags);
+ return;
+ }
+
+ if (fcport->disc_state == DSC_DELETE_PEND) {
+ spin_unlock_irqrestore(&ha->tgt.sess_lock, flags);
+ return;
+ }
+
+ if (!sess->se_sess) {
+ spin_unlock_irqrestore(&ha->tgt.sess_lock, flags);
+
+ mutex_lock(&vha->vha_tgt.tgt_mutex);
+ sess = qlt_create_sess(vha, fcport, false);
+ mutex_unlock(&vha->vha_tgt.tgt_mutex);
+
+ spin_lock_irqsave(&ha->tgt.sess_lock, flags);
+ } else {
+ if (fcport->fw_login_state == DSC_LS_PRLI_COMP) {
+ spin_unlock_irqrestore(&ha->tgt.sess_lock, flags);
+ return;
+ }
+
+ if (!kref_get_unless_zero(&sess->sess_kref)) {
+ ql_dbg(ql_dbg_disc, vha, 0xffff,
+ "%s: kref_get fail sess %8phC \n",
+ __func__, sess->port_name);
+ spin_unlock_irqrestore(&ha->tgt.sess_lock, flags);
+ return;
+ }
+
+ ql_dbg(ql_dbg_tgt_mgt, vha, 0xf04c,
+ "qla_target(%u): %ssession for port %8phC "
+ "(loop ID %d) reappeared\n", vha->vp_idx,
+ sess->local ? "local " : "", sess->port_name, sess->loop_id);
+
+ ql_dbg(ql_dbg_tgt_mgt, vha, 0xf007,
+ "Reappeared sess %p\n", sess);
+
+ ha->tgt.tgt_ops->update_sess(sess, fcport->d_id,
+ fcport->loop_id,
+ (fcport->flags & FCF_CONF_COMP_SUPPORTED));
+ }
+
+ if (sess && sess->local) {
+ ql_dbg(ql_dbg_tgt_mgt, vha, 0xf04d,
+ "qla_target(%u): local session for "
+ "port %8phC (loop ID %d) became global\n", vha->vp_idx,
+ fcport->port_name, sess->loop_id);
+ sess->local = 0;
+ }
+ ha->tgt.tgt_ops->put_sess(sess);
+ spin_unlock_irqrestore(&ha->tgt.sess_lock, flags);
+}
/*
* This is a zero-base ref-counting solution, since hardware_lock
* guarantees that ref_count is not modified concurrently.
* Upon successful return content of iocb is undefined
*/
-static qlt_plogi_ack_t *
+static struct qlt_plogi_ack_t *
qlt_plogi_ack_find_add(struct scsi_qla_host *vha, port_id_t *id,
struct imm_ntfy_from_isp *iocb)
{
- qlt_plogi_ack_t *pla;
+ struct qlt_plogi_ack_t *pla;
list_for_each_entry(pla, &vha->plogi_ack_list, list) {
if (pla->id.b24 == id->b24) {
qlt_send_term_imm_notif(vha, &pla->iocb, 1);
- pla->iocb = *iocb;
+ memcpy(&pla->iocb, iocb, sizeof(pla->iocb));
return pla;
}
}
return NULL;
}
- pla->iocb = *iocb;
+ memcpy(&pla->iocb, iocb, sizeof(pla->iocb));
pla->id = *id;
list_add_tail(&pla->list, &vha->plogi_ack_list);
return pla;
}
-static void qlt_plogi_ack_unref(struct scsi_qla_host *vha, qlt_plogi_ack_t *pla)
+void qlt_plogi_ack_unref(struct scsi_qla_host *vha,
+ struct qlt_plogi_ack_t *pla)
{
+ struct imm_ntfy_from_isp *iocb = &pla->iocb;
+ port_id_t port_id;
+ uint16_t loop_id;
+ fc_port_t *fcport = pla->fcport;
+
BUG_ON(!pla->ref_count);
pla->ref_count--;
if (pla->ref_count)
return;
- ql_dbg(ql_dbg_async, vha, 0x5089,
+ ql_dbg(ql_dbg_disc, vha, 0x5089,
"Sending PLOGI ACK to wwn %8phC s_id %02x:%02x:%02x loop_id %#04x"
- " exch %#x ox_id %#x\n", pla->iocb.u.isp24.port_name,
- pla->iocb.u.isp24.port_id[2], pla->iocb.u.isp24.port_id[1],
- pla->iocb.u.isp24.port_id[0],
- le16_to_cpu(pla->iocb.u.isp24.nport_handle),
- pla->iocb.u.isp24.exchange_address, pla->iocb.ox_id);
- qlt_send_notify_ack(vha, &pla->iocb, 0, 0, 0, 0, 0, 0);
+ " exch %#x ox_id %#x\n", iocb->u.isp24.port_name,
+ iocb->u.isp24.port_id[2], iocb->u.isp24.port_id[1],
+ iocb->u.isp24.port_id[0],
+ le16_to_cpu(iocb->u.isp24.nport_handle),
+ iocb->u.isp24.exchange_address, iocb->ox_id);
+
+ port_id.b.domain = iocb->u.isp24.port_id[2];
+ port_id.b.area = iocb->u.isp24.port_id[1];
+ port_id.b.al_pa = iocb->u.isp24.port_id[0];
+ port_id.b.rsvd_1 = 0;
+
+ loop_id = le16_to_cpu(iocb->u.isp24.nport_handle);
+
+ fcport->loop_id = loop_id;
+ fcport->d_id = port_id;
+ qla24xx_post_nack_work(vha, fcport, iocb, SRB_NACK_PLOGI);
+
+ list_for_each_entry(fcport, &vha->vp_fcports, list) {
+ if (fcport->plogi_link[QLT_PLOGI_LINK_SAME_WWN] == pla)
+ fcport->plogi_link[QLT_PLOGI_LINK_SAME_WWN] = NULL;
+ if (fcport->plogi_link[QLT_PLOGI_LINK_CONFLICT] == pla)
+ fcport->plogi_link[QLT_PLOGI_LINK_CONFLICT] = NULL;
+ }
list_del(&pla->list);
kmem_cache_free(qla_tgt_plogi_cachep, pla);
}
-static void
-qlt_plogi_ack_link(struct scsi_qla_host *vha, qlt_plogi_ack_t *pla,
- struct qla_tgt_sess *sess, qlt_plogi_link_t link)
+void
+qlt_plogi_ack_link(struct scsi_qla_host *vha, struct qlt_plogi_ack_t *pla,
+ struct fc_port *sess, enum qlt_plogi_link_t link)
{
+ struct imm_ntfy_from_isp *iocb = &pla->iocb;
/* Inc ref_count first because link might already be pointing at pla */
pla->ref_count++;
+ ql_dbg(ql_dbg_tgt_mgt, vha, 0xf097,
+ "Linking sess %p [%d] wwn %8phC with PLOGI ACK to wwn %8phC"
+ " s_id %02x:%02x:%02x, ref=%d pla %p link %d\n",
+ sess, link, sess->port_name,
+ iocb->u.isp24.port_name, iocb->u.isp24.port_id[2],
+ iocb->u.isp24.port_id[1], iocb->u.isp24.port_id[0],
+ pla->ref_count, pla, link);
+
if (sess->plogi_link[link])
qlt_plogi_ack_unref(vha, sess->plogi_link[link]);
- ql_dbg(ql_dbg_tgt_mgt, vha, 0xf097,
- "Linking sess %p [%d] wwn %8phC with PLOGI ACK to wwn %8phC"
- " s_id %02x:%02x:%02x, ref=%d\n", sess, link, sess->port_name,
- pla->iocb.u.isp24.port_name, pla->iocb.u.isp24.port_id[2],
- pla->iocb.u.isp24.port_id[1], pla->iocb.u.isp24.port_id[0],
- pla->ref_count);
+ if (link == QLT_PLOGI_LINK_SAME_WWN)
+ pla->fcport = sess;
sess->plogi_link[link] = pla;
}
static void qlt_free_session_done(struct work_struct *work)
{
- struct qla_tgt_sess *sess = container_of(work, struct qla_tgt_sess,
+ struct fc_port *sess = container_of(work, struct fc_port,
free_work);
struct qla_tgt *tgt = sess->tgt;
struct scsi_qla_host *vha = sess->vha;
struct qla_hw_data *ha = vha->hw;
unsigned long flags;
bool logout_started = false;
- fc_port_t fcport;
+ struct event_arg ea;
+ scsi_qla_host_t *base_vha;
ql_dbg(ql_dbg_tgt_mgt, vha, 0xf084,
"%s: se_sess %p / sess %p from port %8phC loop_id %#04x"
" s_id %02x:%02x:%02x logout %d keep %d els_logo %d\n",
__func__, sess->se_sess, sess, sess->port_name, sess->loop_id,
- sess->s_id.b.domain, sess->s_id.b.area, sess->s_id.b.al_pa,
+ sess->d_id.b.domain, sess->d_id.b.area, sess->d_id.b.al_pa,
sess->logout_on_delete, sess->keep_nport_handle,
sess->send_els_logo);
- BUG_ON(!tgt);
- if (sess->send_els_logo) {
- qlt_port_logo_t logo;
- logo.id = sess->s_id;
- logo.cmd_count = 0;
- qlt_send_first_logo(vha, &logo);
- }
+ if (!IS_SW_RESV_ADDR(sess->d_id)) {
+ if (sess->send_els_logo) {
+ qlt_port_logo_t logo;
- if (sess->logout_on_delete) {
- int rc;
+ logo.id = sess->d_id;
+ logo.cmd_count = 0;
+ qlt_send_first_logo(vha, &logo);
+ }
- memset(&fcport, 0, sizeof(fcport));
- fcport.loop_id = sess->loop_id;
- fcport.d_id = sess->s_id;
- memcpy(fcport.port_name, sess->port_name, WWN_SIZE);
- fcport.vha = vha;
- fcport.tgt_session = sess;
-
- rc = qla2x00_post_async_logout_work(vha, &fcport, NULL);
- if (rc != QLA_SUCCESS)
- ql_log(ql_log_warn, vha, 0xf085,
- "Schedule logo failed sess %p rc %d\n",
- sess, rc);
- else
- logout_started = true;
+ if (sess->logout_on_delete) {
+ int rc;
+
+ rc = qla2x00_post_async_logout_work(vha, sess, NULL);
+ if (rc != QLA_SUCCESS)
+ ql_log(ql_log_warn, vha, 0xf085,
+ "Schedule logo failed sess %p rc %d\n",
+ sess, rc);
+ else
+ logout_started = true;
+ }
}
/*
msleep(100);
}
- ql_dbg(ql_dbg_tgt_mgt, vha, 0xf087,
- "%s: sess %p logout completed\n",
- __func__, sess);
+ ql_dbg(ql_dbg_disc, vha, 0xf087,
+ "%s: sess %p logout completed\n",__func__, sess);
}
- spin_lock_irqsave(&ha->hardware_lock, flags);
+ if (sess->logo_ack_needed) {
+ sess->logo_ack_needed = 0;
+ qla24xx_async_notify_ack(vha, sess,
+ (struct imm_ntfy_from_isp *)sess->iocb, SRB_NACK_LOGO);
+ }
+
+ spin_lock_irqsave(&ha->tgt.sess_lock, flags);
+ if (sess->se_sess) {
+ sess->se_sess = NULL;
+ if (tgt && !IS_SW_RESV_ADDR(sess->d_id))
+ tgt->sess_count--;
+ }
+
+ sess->disc_state = DSC_DELETED;
+ sess->fw_login_state = DSC_LS_PORT_UNAVAIL;
+ sess->deleted = QLA_SESS_DELETED;
+ sess->login_retry = vha->hw->login_retry_count;
+
+ if (sess->login_succ && !IS_SW_RESV_ADDR(sess->d_id)) {
+ vha->fcport_count--;
+ sess->login_succ = 0;
+ }
+
+ if (sess->chip_reset != sess->vha->hw->chip_reset)
+ qla2x00_clear_loop_id(sess);
+
+ if (sess->conflict) {
+ sess->conflict->login_pause = 0;
+ sess->conflict = NULL;
+ if (!test_bit(UNLOADING, &vha->dpc_flags))
+ set_bit(RELOGIN_NEEDED, &vha->dpc_flags);
+ }
{
- qlt_plogi_ack_t *own =
+ struct qlt_plogi_ack_t *own =
sess->plogi_link[QLT_PLOGI_LINK_SAME_WWN];
- qlt_plogi_ack_t *con =
+ struct qlt_plogi_ack_t *con =
sess->plogi_link[QLT_PLOGI_LINK_CONFLICT];
+ struct imm_ntfy_from_isp *iocb;
if (con) {
+ iocb = &con->iocb;
ql_dbg(ql_dbg_tgt_mgt, vha, 0xf099,
- "se_sess %p / sess %p port %8phC is gone,"
- " %s (ref=%d), releasing PLOGI for %8phC (ref=%d)\n",
- sess->se_sess, sess, sess->port_name,
- own ? "releasing own PLOGI" :
- "no own PLOGI pending",
- own ? own->ref_count : -1,
- con->iocb.u.isp24.port_name, con->ref_count);
+ "se_sess %p / sess %p port %8phC is gone,"
+ " %s (ref=%d), releasing PLOGI for %8phC (ref=%d)\n",
+ sess->se_sess, sess, sess->port_name,
+ own ? "releasing own PLOGI" : "no own PLOGI pending",
+ own ? own->ref_count : -1,
+ iocb->u.isp24.port_name, con->ref_count);
qlt_plogi_ack_unref(vha, con);
+ sess->plogi_link[QLT_PLOGI_LINK_CONFLICT] = NULL;
} else {
ql_dbg(ql_dbg_tgt_mgt, vha, 0xf09a,
"se_sess %p / sess %p port %8phC is gone, %s (ref=%d)\n",
own ? own->ref_count : -1);
}
- if (own)
+ if (own) {
+ sess->fw_login_state = DSC_LS_PLOGI_PEND;
qlt_plogi_ack_unref(vha, own);
+ sess->plogi_link[QLT_PLOGI_LINK_SAME_WWN] = NULL;
+ }
}
-
- list_del(&sess->sess_list_entry);
-
- spin_unlock_irqrestore(&ha->hardware_lock, flags);
+ spin_unlock_irqrestore(&ha->tgt.sess_lock, flags);
ql_dbg(ql_dbg_tgt_mgt, vha, 0xf001,
- "Unregistration of sess %p finished\n", sess);
+ "Unregistration of sess %p %8phC finished fcp_cnt %d\n",
+ sess, sess->port_name, vha->fcport_count);
- kfree(sess);
- /*
- * We need to protect against race, when tgt is freed before or
- * inside wake_up()
- */
- tgt->sess_count--;
- if (tgt->sess_count == 0)
+ if (tgt && (tgt->sess_count == 0))
wake_up_all(&tgt->waitQ);
+
+ if (vha->fcport_count == 0)
+ wake_up_all(&vha->fcport_waitQ);
+
+ base_vha = pci_get_drvdata(ha->pdev);
+ if (test_bit(PFLG_DRIVER_REMOVING, &base_vha->pci_flags))
+ return;
+
+ if (!tgt || !tgt->tgt_stop) {
+ memset(&ea, 0, sizeof(ea));
+ ea.event = FCME_DELETE_DONE;
+ ea.fcport = sess;
+ qla2x00_fcport_event_handler(vha, &ea);
+ }
}
/* ha->tgt.sess_lock supposed to be held on entry */
-static void qlt_release_session(struct kref *kref)
+void qlt_unreg_sess(struct fc_port *sess)
{
- struct qla_tgt_sess *sess =
- container_of(kref, struct qla_tgt_sess, sess_kref);
struct scsi_qla_host *vha = sess->vha;
+ ql_dbg(ql_dbg_disc, sess->vha, 0xffff,
+ "%s sess %p for deletion %8phC\n",
+ __func__, sess, sess->port_name);
+
if (sess->se_sess)
vha->hw->tgt.tgt_ops->clear_nacl_from_fcport_map(sess);
- if (!list_empty(&sess->del_list_entry))
- list_del_init(&sess->del_list_entry);
+ qla2x00_mark_device_lost(vha, sess, 1, 1);
+
sess->deleted = QLA_SESS_DELETION_IN_PROGRESS;
+ sess->disc_state = DSC_DELETE_PEND;
+ sess->last_rscn_gen = sess->rscn_gen;
+ sess->last_login_gen = sess->login_gen;
INIT_WORK(&sess->free_work, qlt_free_session_done);
schedule_work(&sess->free_work);
}
-
-void qlt_put_sess(struct qla_tgt_sess *sess)
-{
- if (!sess)
- return;
-
- assert_spin_locked(&sess->vha->hw->tgt.sess_lock);
- kref_put(&sess->sess_kref, qlt_release_session);
-}
-EXPORT_SYMBOL(qlt_put_sess);
+EXPORT_SYMBOL(qlt_unreg_sess);
static int qlt_reset(struct scsi_qla_host *vha, void *iocb, int mcmd)
{
struct qla_hw_data *ha = vha->hw;
- struct qla_tgt_sess *sess = NULL;
+ struct fc_port *sess = NULL;
uint16_t loop_id;
int res = 0;
struct imm_ntfy_from_isp *n = (struct imm_ntfy_from_isp *)iocb;
spin_lock_irqsave(&ha->tgt.sess_lock, flags);
qlt_clear_tgt_db(vha->vha_tgt.qla_tgt);
spin_unlock_irqrestore(&ha->tgt.sess_lock, flags);
-#if 0 /* FIXME: do we need to choose a session here? */
- if (!list_empty(&ha->tgt.qla_tgt->sess_list)) {
- sess = list_entry(ha->tgt.qla_tgt->sess_list.next,
- typeof(*sess), sess_list_entry);
- switch (mcmd) {
- case QLA_TGT_NEXUS_LOSS_SESS:
- mcmd = QLA_TGT_NEXUS_LOSS;
- break;
- case QLA_TGT_ABORT_ALL_SESS:
- mcmd = QLA_TGT_ABORT_ALL;
- break;
- case QLA_TGT_NEXUS_LOSS:
- case QLA_TGT_ABORT_ALL:
- break;
- default:
- ql_dbg(ql_dbg_tgt, vha, 0xe046,
- "qla_target(%d): Not allowed "
- "command %x in %s", vha->vp_idx,
- mcmd, __func__);
- sess = NULL;
- break;
- }
- } else
- sess = NULL;
-#endif
} else {
spin_lock_irqsave(&ha->tgt.sess_lock, flags);
sess = ha->tgt.tgt_ops->find_sess_by_loop_id(vha, loop_id);
return qlt_issue_task_mgmt(sess, 0, mcmd, iocb, QLA24XX_MGMT_SEND_NACK);
}
+static void qla24xx_chk_fcp_state(struct fc_port *sess)
+{
+ if (sess->chip_reset != sess->vha->hw->chip_reset) {
+ sess->logout_on_delete = 0;
+ sess->logo_ack_needed = 0;
+ sess->fw_login_state = DSC_LS_PORT_UNAVAIL;
+ sess->scan_state = 0;
+ }
+}
+
/* ha->tgt.sess_lock supposed to be held on entry */
-static void qlt_schedule_sess_for_deletion(struct qla_tgt_sess *sess,
+void qlt_schedule_sess_for_deletion(struct fc_port *sess,
bool immediate)
{
struct qla_tgt *tgt = sess->tgt;
- uint32_t dev_loss_tmo = tgt->ha->port_down_retry_count + 5;
- if (sess->deleted) {
- /* Upgrade to unconditional deletion in case it was temporary */
- if (immediate && sess->deleted == QLA_SESS_DELETION_PENDING)
- list_del(&sess->del_list_entry);
- else
+ if (sess->disc_state == DSC_DELETE_PEND)
+ return;
+
+ if (sess->disc_state == DSC_DELETED) {
+ if (tgt && tgt->tgt_stop && (tgt->sess_count == 0))
+ wake_up_all(&tgt->waitQ);
+ if (sess->vha->fcport_count == 0)
+ wake_up_all(&sess->vha->fcport_waitQ);
+
+ if (!sess->plogi_link[QLT_PLOGI_LINK_SAME_WWN] &&
+ !sess->plogi_link[QLT_PLOGI_LINK_CONFLICT])
return;
}
- ql_dbg(ql_dbg_tgt, sess->vha, 0xe001,
- "Scheduling sess %p for deletion\n", sess);
+ sess->disc_state = DSC_DELETE_PEND;
- if (immediate) {
- dev_loss_tmo = 0;
- sess->deleted = QLA_SESS_DELETION_IN_PROGRESS;
- list_add(&sess->del_list_entry, &tgt->del_sess_list);
- } else {
- sess->deleted = QLA_SESS_DELETION_PENDING;
- list_add_tail(&sess->del_list_entry, &tgt->del_sess_list);
- }
+ if (sess->deleted == QLA_SESS_DELETED)
+ sess->logout_on_delete = 0;
- sess->expires = jiffies + dev_loss_tmo * HZ;
+ sess->deleted = QLA_SESS_DELETION_IN_PROGRESS;
+ qla24xx_chk_fcp_state(sess);
- ql_dbg(ql_dbg_tgt, sess->vha, 0xe048,
- "qla_target(%d): session for port %8phC (loop ID %d s_id %02x:%02x:%02x)"
- " scheduled for deletion in %u secs (expires: %lu) immed: %d, logout: %d, gen: %#x\n",
- sess->vha->vp_idx, sess->port_name, sess->loop_id,
- sess->s_id.b.domain, sess->s_id.b.area, sess->s_id.b.al_pa,
- dev_loss_tmo, sess->expires, immediate, sess->logout_on_delete,
- sess->generation);
+ ql_dbg(ql_dbg_tgt, sess->vha, 0xe001,
+ "Scheduling sess %p for deletion\n", sess);
- if (immediate)
- mod_delayed_work(system_wq, &tgt->sess_del_work, 0);
- else
- schedule_delayed_work(&tgt->sess_del_work,
- sess->expires - jiffies);
+ schedule_work(&sess->del_work);
+}
+
+void qlt_schedule_sess_for_deletion_lock(struct fc_port *sess)
+{
+ unsigned long flags;
+ struct qla_hw_data *ha = sess->vha->hw;
+ spin_lock_irqsave(&ha->tgt.sess_lock, flags);
+ qlt_schedule_sess_for_deletion(sess, 1);
+ spin_unlock_irqrestore(&ha->tgt.sess_lock, flags);
}
/* ha->tgt.sess_lock supposed to be held on entry */
static void qlt_clear_tgt_db(struct qla_tgt *tgt)
{
- struct qla_tgt_sess *sess;
+ struct fc_port *sess;
+ scsi_qla_host_t *vha = tgt->vha;
- list_for_each_entry(sess, &tgt->sess_list, sess_list_entry)
- qlt_schedule_sess_for_deletion(sess, true);
+ list_for_each_entry(sess, &vha->vp_fcports, list) {
+ if (sess->se_sess)
+ qlt_schedule_sess_for_deletion(sess, 1);
+ }
/* At this point tgt could be already dead */
}
goto out_free_id_list;
}
- id_iter = (char *)gid_list;
- res = -ENOENT;
- for (i = 0; i < entries; i++) {
- struct gid_list_info *gid = (struct gid_list_info *)id_iter;
- if ((gid->al_pa == s_id[2]) &&
- (gid->area == s_id[1]) &&
- (gid->domain == s_id[0])) {
- *loop_id = le16_to_cpu(gid->loop_id);
- res = 0;
- break;
- }
- id_iter += ha->gid_list_info_size;
- }
-
-out_free_id_list:
- dma_free_coherent(&ha->pdev->dev, qla2x00_gid_list_size(ha),
- gid_list, gid_list_dma);
- return res;
-}
-
-/* ha->tgt.sess_lock supposed to be held on entry */
-static void qlt_undelete_sess(struct qla_tgt_sess *sess)
-{
- BUG_ON(sess->deleted != QLA_SESS_DELETION_PENDING);
-
- list_del_init(&sess->del_list_entry);
- sess->deleted = 0;
-}
-
-static void qlt_del_sess_work_fn(struct delayed_work *work)
-{
- struct qla_tgt *tgt = container_of(work, struct qla_tgt,
- sess_del_work);
- struct scsi_qla_host *vha = tgt->vha;
- struct qla_hw_data *ha = vha->hw;
- struct qla_tgt_sess *sess;
- unsigned long flags, elapsed;
-
- spin_lock_irqsave(&ha->tgt.sess_lock, flags);
- while (!list_empty(&tgt->del_sess_list)) {
- sess = list_entry(tgt->del_sess_list.next, typeof(*sess),
- del_list_entry);
- elapsed = jiffies;
- if (time_after_eq(elapsed, sess->expires)) {
- /* No turning back */
- list_del_init(&sess->del_list_entry);
- sess->deleted = QLA_SESS_DELETION_IN_PROGRESS;
-
- ql_dbg(ql_dbg_tgt_mgt, vha, 0xf004,
- "Timeout: sess %p about to be deleted\n",
- sess);
- if (sess->se_sess)
- ha->tgt.tgt_ops->shutdown_sess(sess);
- qlt_put_sess(sess);
- } else {
- schedule_delayed_work(&tgt->sess_del_work,
- sess->expires - elapsed);
+ id_iter = (char *)gid_list;
+ res = -ENOENT;
+ for (i = 0; i < entries; i++) {
+ struct gid_list_info *gid = (struct gid_list_info *)id_iter;
+ if ((gid->al_pa == s_id[2]) &&
+ (gid->area == s_id[1]) &&
+ (gid->domain == s_id[0])) {
+ *loop_id = le16_to_cpu(gid->loop_id);
+ res = 0;
break;
}
+ id_iter += ha->gid_list_info_size;
}
- spin_unlock_irqrestore(&ha->tgt.sess_lock, flags);
+
+out_free_id_list:
+ dma_free_coherent(&ha->pdev->dev, qla2x00_gid_list_size(ha),
+ gid_list, gid_list_dma);
+ return res;
}
/*
* Adds an extra ref to allow to drop hw lock after adding sess to the list.
* Caller must put it.
*/
-static struct qla_tgt_sess *qlt_create_sess(
+static struct fc_port *qlt_create_sess(
struct scsi_qla_host *vha,
fc_port_t *fcport,
bool local)
{
struct qla_hw_data *ha = vha->hw;
- struct qla_tgt_sess *sess;
+ struct fc_port *sess = fcport;
unsigned long flags;
- /* Check to avoid double sessions */
- spin_lock_irqsave(&ha->tgt.sess_lock, flags);
- list_for_each_entry(sess, &vha->vha_tgt.qla_tgt->sess_list,
- sess_list_entry) {
- if (!memcmp(sess->port_name, fcport->port_name, WWN_SIZE)) {
- ql_dbg(ql_dbg_tgt_mgt, vha, 0xf005,
- "Double sess %p found (s_id %x:%x:%x, "
- "loop_id %d), updating to d_id %x:%x:%x, "
- "loop_id %d", sess, sess->s_id.b.domain,
- sess->s_id.b.al_pa, sess->s_id.b.area,
- sess->loop_id, fcport->d_id.b.domain,
- fcport->d_id.b.al_pa, fcport->d_id.b.area,
- fcport->loop_id);
-
- /* Cannot undelete at this point */
- if (sess->deleted == QLA_SESS_DELETION_IN_PROGRESS) {
- spin_unlock_irqrestore(&ha->tgt.sess_lock,
- flags);
- return NULL;
- }
-
- if (sess->deleted)
- qlt_undelete_sess(sess);
-
- if (!sess->se_sess) {
- if (ha->tgt.tgt_ops->check_initiator_node_acl(vha,
- &sess->port_name[0], sess) < 0) {
- spin_unlock_irqrestore(&ha->tgt.sess_lock, flags);
- return NULL;
- }
- }
-
- kref_get(&sess->sess_kref);
- ha->tgt.tgt_ops->update_sess(sess, fcport->d_id, fcport->loop_id,
- (fcport->flags & FCF_CONF_COMP_SUPPORTED));
-
- if (sess->local && !local)
- sess->local = 0;
-
- qlt_do_generation_tick(vha, &sess->generation);
-
- spin_unlock_irqrestore(&ha->tgt.sess_lock, flags);
+ if (vha->vha_tgt.qla_tgt->tgt_stop)
+ return NULL;
- return sess;
+ if (fcport->se_sess) {
+ if (!kref_get_unless_zero(&sess->sess_kref)) {
+ ql_dbg(ql_dbg_disc, vha, 0xffff,
+ "%s: kref_get_unless_zero failed for %8phC\n",
+ __func__, sess->port_name);
+ return NULL;
}
- }
- spin_unlock_irqrestore(&ha->tgt.sess_lock, flags);
-
- sess = kzalloc(sizeof(*sess), GFP_KERNEL);
- if (!sess) {
- ql_dbg(ql_dbg_tgt_mgt, vha, 0xf04a,
- "qla_target(%u): session allocation failed, all commands "
- "from port %8phC will be refused", vha->vp_idx,
- fcport->port_name);
-
- return NULL;
+ return fcport;
}
sess->tgt = vha->vha_tgt.qla_tgt;
- sess->vha = vha;
- sess->s_id = fcport->d_id;
- sess->loop_id = fcport->loop_id;
sess->local = local;
- kref_init(&sess->sess_kref);
- INIT_LIST_HEAD(&sess->del_list_entry);
- /* Under normal circumstances we want to logout from firmware when
+ /*
+ * Under normal circumstances we want to logout from firmware when
* session eventually ends and release corresponding nport handle.
* In the exception cases (e.g. when new PLOGI is waiting) corresponding
- * code will adjust these flags as necessary. */
+ * code will adjust these flags as necessary.
+ */
sess->logout_on_delete = 1;
sess->keep_nport_handle = 0;
+ sess->logout_completed = 0;
- ql_dbg(ql_dbg_tgt_mgt, vha, 0xf006,
- "Adding sess %p to tgt %p via ->check_initiator_node_acl()\n",
- sess, vha->vha_tgt.qla_tgt);
-
- sess->conf_compl_supported = (fcport->flags & FCF_CONF_COMP_SUPPORTED);
- BUILD_BUG_ON(sizeof(sess->port_name) != sizeof(fcport->port_name));
- memcpy(sess->port_name, fcport->port_name, sizeof(sess->port_name));
-
- spin_lock_irqsave(&ha->tgt.sess_lock, flags);
- list_add_tail(&sess->sess_list_entry, &vha->vha_tgt.qla_tgt->sess_list);
- vha->vha_tgt.qla_tgt->sess_count++;
- qlt_do_generation_tick(vha, &sess->generation);
- spin_unlock_irqrestore(&ha->tgt.sess_lock, flags);
-
- ql_dbg(ql_dbg_tgt_mgt, vha, 0xf04b,
- "qla_target(%d): %ssession for wwn %8phC (loop_id %d, "
- "s_id %x:%x:%x, confirmed completion %ssupported) added\n",
- vha->vp_idx, local ? "local " : "", fcport->port_name,
- fcport->loop_id, sess->s_id.b.domain, sess->s_id.b.area,
- sess->s_id.b.al_pa, sess->conf_compl_supported ? "" : "not ");
-
- /*
- * Determine if this fc_port->port_name is allowed to access
- * target mode using explict NodeACLs+MappedLUNs, or using
- * TPG demo mode. If this is successful a target mode FC nexus
- * is created.
- */
if (ha->tgt.tgt_ops->check_initiator_node_acl(vha,
&fcport->port_name[0], sess) < 0) {
+ ql_dbg(ql_dbg_tgt_mgt, vha, 0xffff,
+ "(%d) %8phC check_initiator_node_acl failed\n",
+ vha->vp_idx, fcport->port_name);
return NULL;
} else {
+ kref_init(&fcport->sess_kref);
/*
- * Take an extra reference to ->sess_kref here to handle qla_tgt_sess
- * access across ->tgt.sess_lock reaquire.
+ * Take an extra reference to ->sess_kref here to handle
+ * fc_port access across ->tgt.sess_lock reaquire.
*/
- kref_get(&sess->sess_kref);
- }
-
- return sess;
-}
-
-/*
- * Called from qla2x00_reg_remote_port()
- */
-void qlt_fc_port_added(struct scsi_qla_host *vha, fc_port_t *fcport)
-{
- struct qla_hw_data *ha = vha->hw;
- struct qla_tgt *tgt = vha->vha_tgt.qla_tgt;
- struct qla_tgt_sess *sess;
- unsigned long flags;
-
- if (!vha->hw->tgt.tgt_ops)
- return;
-
- if (!tgt || (fcport->port_type != FCT_INITIATOR))
- return;
+ if (!kref_get_unless_zero(&sess->sess_kref)) {
+ ql_dbg(ql_dbg_disc, vha, 0xffff,
+ "%s: kref_get_unless_zero failed for %8phC\n",
+ __func__, sess->port_name);
+ return NULL;
+ }
- if (qla_ini_mode_enabled(vha))
- return;
+ spin_lock_irqsave(&ha->tgt.sess_lock, flags);
+ if (!IS_SW_RESV_ADDR(sess->d_id))
+ vha->vha_tgt.qla_tgt->sess_count++;
- spin_lock_irqsave(&ha->tgt.sess_lock, flags);
- if (tgt->tgt_stop) {
+ qlt_do_generation_tick(vha, &sess->generation);
spin_unlock_irqrestore(&ha->tgt.sess_lock, flags);
- return;
}
- sess = qlt_find_sess_by_port_name(tgt, fcport->port_name);
- if (!sess) {
- spin_unlock_irqrestore(&ha->tgt.sess_lock, flags);
-
- mutex_lock(&vha->vha_tgt.tgt_mutex);
- sess = qlt_create_sess(vha, fcport, false);
- mutex_unlock(&vha->vha_tgt.tgt_mutex);
-
- spin_lock_irqsave(&ha->tgt.sess_lock, flags);
- } else if (sess->deleted == QLA_SESS_DELETION_IN_PROGRESS) {
- /* Point of no return */
- spin_unlock_irqrestore(&ha->tgt.sess_lock, flags);
- return;
- } else {
- kref_get(&sess->sess_kref);
-
- if (sess->deleted) {
- qlt_undelete_sess(sess);
- ql_dbg(ql_dbg_tgt_mgt, vha, 0xf04c,
- "qla_target(%u): %ssession for port %8phC "
- "(loop ID %d) reappeared\n", vha->vp_idx,
- sess->local ? "local " : "", sess->port_name,
- sess->loop_id);
+ ql_dbg(ql_dbg_tgt_mgt, vha, 0xf006,
+ "Adding sess %p se_sess %p to tgt %p sess_count %d\n",
+ sess, sess->se_sess, vha->vha_tgt.qla_tgt,
+ vha->vha_tgt.qla_tgt->sess_count);
- ql_dbg(ql_dbg_tgt_mgt, vha, 0xf007,
- "Reappeared sess %p\n", sess);
- }
- ha->tgt.tgt_ops->update_sess(sess, fcport->d_id, fcport->loop_id,
- (fcport->flags & FCF_CONF_COMP_SUPPORTED));
- }
+ ql_dbg(ql_dbg_tgt_mgt, vha, 0xf04b,
+ "qla_target(%d): %ssession for wwn %8phC (loop_id %d, "
+ "s_id %x:%x:%x, confirmed completion %ssupported) added\n",
+ vha->vp_idx, local ? "local " : "", fcport->port_name,
+ fcport->loop_id, sess->d_id.b.domain, sess->d_id.b.area,
+ sess->d_id.b.al_pa, sess->conf_compl_supported ? "" : "not ");
- if (sess && sess->local) {
- ql_dbg(ql_dbg_tgt_mgt, vha, 0xf04d,
- "qla_target(%u): local session for "
- "port %8phC (loop ID %d) became global\n", vha->vp_idx,
- fcport->port_name, sess->loop_id);
- sess->local = 0;
- }
- qlt_put_sess(sess);
- spin_unlock_irqrestore(&ha->tgt.sess_lock, flags);
+ return sess;
}
/*
qlt_fc_port_deleted(struct scsi_qla_host *vha, fc_port_t *fcport, int max_gen)
{
struct qla_tgt *tgt = vha->vha_tgt.qla_tgt;
- struct qla_tgt_sess *sess;
+ struct fc_port *sess = fcport;
unsigned long flags;
if (!vha->hw->tgt.tgt_ops)
spin_unlock_irqrestore(&vha->hw->tgt.sess_lock, flags);
return;
}
- sess = qlt_find_sess_by_port_name(tgt, fcport->port_name);
- if (!sess) {
+ if (!sess->se_sess) {
spin_unlock_irqrestore(&vha->hw->tgt.sess_lock, flags);
return;
}
* We need to protect against race, when tgt is freed before or
* inside wake_up()
*/
- spin_lock_irqsave(&ha->hardware_lock, flags);
+ spin_lock_irqsave(&ha->tgt.sess_lock, flags);
ql_dbg(ql_dbg_tgt, tgt->vha, 0xe002,
- "tgt %p, empty(sess_list)=%d sess_count=%d\n",
- tgt, list_empty(&tgt->sess_list), tgt->sess_count);
+ "tgt %p, sess_count=%d\n",
+ tgt, tgt->sess_count);
res = (tgt->sess_count == 0);
- spin_unlock_irqrestore(&ha->hardware_lock, flags);
+ spin_unlock_irqrestore(&ha->tgt.sess_lock, flags);
return res;
}
mutex_unlock(&vha->vha_tgt.tgt_mutex);
mutex_unlock(&qla_tgt_mutex);
- flush_delayed_work(&tgt->sess_del_work);
-
ql_dbg(ql_dbg_tgt_mgt, vha, 0xf009,
"Waiting for sess works (tgt %p)", tgt);
spin_lock_irqsave(&tgt->sess_work_lock, flags);
spin_unlock_irqrestore(&tgt->sess_work_lock, flags);
ql_dbg(ql_dbg_tgt_mgt, vha, 0xf00a,
- "Waiting for tgt %p: list_empty(sess_list)=%d "
- "sess_count=%d\n", tgt, list_empty(&tgt->sess_list),
- tgt->sess_count);
+ "Waiting for tgt %p: sess_count=%d\n", tgt, tgt->sess_count);
wait_event(tgt->waitQ, test_tgt_sess_count(tgt));
/* Big hammer */
- if (!ha->flags.host_shutting_down && qla_tgt_mode_enabled(vha))
+ if (!ha->flags.host_shutting_down &&
+ (qla_tgt_mode_enabled(vha) || qla_dual_mode_enabled(vha)))
qlt_disable_vha(vha);
/* Wait for sessions to clear out (just in case) */
nack = (struct nack_to_isp *)pkt;
nack->ox_id = ntfy->ox_id;
+ nack->u.isp24.handle = QLA_TGT_SKIP_HANDLE;
nack->u.isp24.nport_handle = ntfy->u.isp24.nport_handle;
if (le16_to_cpu(ntfy->u.isp24.status) == IMM_NTFY_ELS) {
nack->u.isp24.flags = ntfy->u.isp24.flags &
}
}
+ list_for_each_entry(op, &vha->unknown_atio_list, cmd_list) {
+ if (tag == op->atio.u.isp24.exchange_addr) {
+ op->aborted = true;
+ spin_unlock(&vha->cmd_list_lock);
+ return 1;
+ }
+ }
+
list_for_each_entry(cmd, &vha->qla_cmd_list, cmd_list) {
if (tag == cmd->atio.u.isp24.exchange_addr) {
cmd->aborted = 1;
if (op_key == key && op_lun == lun)
op->aborted = true;
}
+
+ list_for_each_entry(op, &vha->unknown_atio_list, cmd_list) {
+ uint32_t op_key;
+ u64 op_lun;
+
+ op_key = sid_to_key(op->atio.u.isp24.fcp_hdr.s_id);
+ op_lun = scsilun_to_int(
+ (struct scsi_lun *)&op->atio.u.isp24.fcp_cmnd.lun);
+ if (op_key == key && op_lun == lun)
+ op->aborted = true;
+ }
+
list_for_each_entry(cmd, &vha->qla_cmd_list, cmd_list) {
uint32_t cmd_key;
uint32_t cmd_lun;
/* ha->hardware_lock supposed to be held on entry */
static int __qlt_24xx_handle_abts(struct scsi_qla_host *vha,
- struct abts_recv_from_24xx *abts, struct qla_tgt_sess *sess)
+ struct abts_recv_from_24xx *abts, struct fc_port *sess)
{
struct qla_hw_data *ha = vha->hw;
struct se_session *se_sess = sess->se_sess;
u32 lun = 0;
int rc;
bool found_lun = false;
+ unsigned long flags;
- spin_lock(&se_sess->sess_cmd_lock);
+ spin_lock_irqsave(&se_sess->sess_cmd_lock, flags);
list_for_each_entry(se_cmd, &se_sess->sess_cmd_list, se_cmd_list) {
struct qla_tgt_cmd *cmd =
container_of(se_cmd, struct qla_tgt_cmd, se_cmd);
break;
}
}
- spin_unlock(&se_sess->sess_cmd_lock);
+ spin_unlock_irqrestore(&se_sess->sess_cmd_lock, flags);
/* cmd not in LIO lists, look in qla list */
if (!found_lun) {
mcmd->sess = sess;
memcpy(&mcmd->orig_iocb.abts, abts, sizeof(mcmd->orig_iocb.abts));
mcmd->reset_count = vha->hw->chip_reset;
+ mcmd->tmr_func = QLA_TGT_ABTS;
- rc = ha->tgt.tgt_ops->handle_tmr(mcmd, lun, TMR_ABORT_TASK,
+ rc = ha->tgt.tgt_ops->handle_tmr(mcmd, lun, mcmd->tmr_func,
abts->exchange_addr_to_abort);
if (rc != 0) {
ql_dbg(ql_dbg_tgt_mgt, vha, 0xf052,
struct abts_recv_from_24xx *abts)
{
struct qla_hw_data *ha = vha->hw;
- struct qla_tgt_sess *sess;
+ struct fc_port *sess;
uint32_t tag = abts->exchange_addr_to_abort;
uint8_t s_id[3];
int rc;
spin_unlock_irqrestore(&ha->tgt.sess_lock, flags);
- if (sess->deleted == QLA_SESS_DELETION_IN_PROGRESS) {
+ if (sess->deleted) {
qlt_24xx_send_abts_resp(vha, abts, FCP_TMF_REJECTED, false);
return;
}
return;
}
- if (mcmd->flags == QLA24XX_MGMT_SEND_NACK)
- qlt_send_notify_ack(vha, &mcmd->orig_iocb.imm_ntfy,
- 0, 0, 0, 0, 0, 0);
- else {
+ if (mcmd->flags == QLA24XX_MGMT_SEND_NACK) {
+ if (mcmd->orig_iocb.imm_ntfy.u.isp24.status_subcode ==
+ ELS_LOGO ||
+ mcmd->orig_iocb.imm_ntfy.u.isp24.status_subcode ==
+ ELS_PRLO ||
+ mcmd->orig_iocb.imm_ntfy.u.isp24.status_subcode ==
+ ELS_TPRLO) {
+ ql_dbg(ql_dbg_disc, vha, 0xffff,
+ "TM response logo %phC status %#x state %#x",
+ mcmd->sess->port_name, mcmd->fc_tm_rsp,
+ mcmd->flags);
+ qlt_schedule_sess_for_deletion_lock(mcmd->sess);
+ } else {
+ qlt_send_notify_ack(vha, &mcmd->orig_iocb.imm_ntfy,
+ 0, 0, 0, 0, 0, 0);
+ }
+ } else {
if (mcmd->orig_iocb.atio.u.raw.entry_type == ABTS_RECV_24XX)
qlt_24xx_send_abts_resp(vha, &mcmd->orig_iocb.abts,
mcmd->fc_tm_rsp, false);
cmd->conf_compl_supported;
}
-#ifdef CONFIG_QLA_TGT_DEBUG_SRR
-/*
- * Original taken from the XFS code
- */
-static unsigned long qlt_srr_random(void)
-{
- static int Inited;
- static unsigned long RandomValue;
- static DEFINE_SPINLOCK(lock);
- /* cycles pseudo-randomly through all values between 1 and 2^31 - 2 */
- register long rv;
- register long lo;
- register long hi;
- unsigned long flags;
-
- spin_lock_irqsave(&lock, flags);
- if (!Inited) {
- RandomValue = jiffies;
- Inited = 1;
- }
- rv = RandomValue;
- hi = rv / 127773;
- lo = rv % 127773;
- rv = 16807 * lo - 2836 * hi;
- if (rv <= 0)
- rv += 2147483647;
- RandomValue = rv;
- spin_unlock_irqrestore(&lock, flags);
- return rv;
-}
-
-static void qlt_check_srr_debug(struct qla_tgt_cmd *cmd, int *xmit_type)
-{
-#if 0 /* This is not a real status packets lost, so it won't lead to SRR */
- if ((*xmit_type & QLA_TGT_XMIT_STATUS) && (qlt_srr_random() % 200)
- == 50) {
- *xmit_type &= ~QLA_TGT_XMIT_STATUS;
- ql_dbg(ql_dbg_tgt_mgt, cmd->vha, 0xf015,
- "Dropping cmd %p (tag %d) status", cmd, se_cmd->tag);
- }
-#endif
- /*
- * It's currently not possible to simulate SRRs for FCP_WRITE without
- * a physical link layer failure, so don't even try here..
- */
- if (cmd->dma_data_direction != DMA_FROM_DEVICE)
- return;
-
- if (qlt_has_data(cmd) && (cmd->sg_cnt > 1) &&
- ((qlt_srr_random() % 100) == 20)) {
- int i, leave = 0;
- unsigned int tot_len = 0;
-
- while (leave == 0)
- leave = qlt_srr_random() % cmd->sg_cnt;
-
- for (i = 0; i < leave; i++)
- tot_len += cmd->sg[i].length;
-
- ql_dbg(ql_dbg_tgt_mgt, cmd->vha, 0xf016,
- "Cutting cmd %p (tag %d) buffer"
- " tail to len %d, sg_cnt %d (cmd->bufflen %d,"
- " cmd->sg_cnt %d)", cmd, se_cmd->tag, tot_len, leave,
- cmd->bufflen, cmd->sg_cnt);
-
- cmd->bufflen = tot_len;
- cmd->sg_cnt = leave;
- }
-
- if (qlt_has_data(cmd) && ((qlt_srr_random() % 100) == 70)) {
- unsigned int offset = qlt_srr_random() % cmd->bufflen;
-
- ql_dbg(ql_dbg_tgt_mgt, cmd->vha, 0xf017,
- "Cutting cmd %p (tag %d) buffer head "
- "to offset %d (cmd->bufflen %d)", cmd, se_cmd->tag, offset,
- cmd->bufflen);
- if (offset == 0)
- *xmit_type &= ~QLA_TGT_XMIT_DATA;
- else if (qlt_set_data_offset(cmd, offset)) {
- ql_dbg(ql_dbg_tgt_mgt, cmd->vha, 0xf018,
- "qlt_set_data_offset() failed (tag %d)", se_cmd->tag);
- }
- }
-}
-#else
-static inline void qlt_check_srr_debug(struct qla_tgt_cmd *cmd, int *xmit_type)
-{}
-#endif
-
static void qlt_24xx_init_ctio_to_isp(struct ctio7_to_24xx *ctio,
struct qla_tgt_prm *prm)
{
int i;
if (qlt_need_explicit_conf(prm->tgt->ha, prm->cmd, 1)) {
- if (prm->cmd->se_cmd.scsi_status != 0) {
+ if ((prm->rq_result & SS_SCSI_STATUS_BYTE) != 0) {
ql_dbg(ql_dbg_tgt, prm->cmd->vha, 0xe017,
"Skipping EXPLICIT_CONFORM and "
"CTIO7_FLAGS_CONFORM_REQ for FCP READ w/ "
int res;
spin_lock_irqsave(&ha->hardware_lock, flags);
- if (cmd->sess && cmd->sess->deleted == QLA_SESS_DELETION_IN_PROGRESS) {
+ if (cmd->sess && cmd->sess->deleted) {
cmd->state = QLA_TGT_STATE_PROCESSED;
if (cmd->sess->logout_completed)
/* no need to terminate. FW already freed exchange. */
spin_unlock_irqrestore(&ha->hardware_lock, flags);
memset(&prm, 0, sizeof(prm));
- qlt_check_srr_debug(cmd, &xmit_type);
ql_dbg(ql_dbg_tgt, cmd->vha, 0xe018,
"is_send_status=%d, cmd->bufflen=%d, cmd->sg_cnt=%d, cmd->dma_data_direction=%d se_cmd[%p]\n",
spin_lock_irqsave(&ha->hardware_lock, flags);
if (!vha->flags.online || (cmd->reset_count != ha->chip_reset) ||
- (cmd->sess && cmd->sess->deleted == QLA_SESS_DELETION_IN_PROGRESS)) {
+ (cmd->sess && cmd->sess->deleted)) {
/*
* Either the port is not online or this request was from
* previous life, just abort the processing.
return EIO;
}
cmd->aborted = 1;
- cmd->cmd_flags |= BIT_6;
+ cmd->trc_flags |= TRC_ABORT;
spin_unlock_irqrestore(&cmd->cmd_lock, flags);
qlt_send_term_exchange(vha, cmd, &cmd->atio, 0, 1);
void qlt_free_cmd(struct qla_tgt_cmd *cmd)
{
- struct qla_tgt_sess *sess = cmd->sess;
+ struct fc_port *sess = cmd->sess;
ql_dbg(ql_dbg_tgt, cmd->vha, 0xe074,
"%s: se_cmd[%p] ox_id %04x\n",
}
EXPORT_SYMBOL(qlt_free_cmd);
-/* ha->hardware_lock supposed to be held on entry */
-static int qlt_prepare_srr_ctio(struct scsi_qla_host *vha,
- struct qla_tgt_cmd *cmd, void *ctio)
-{
- struct qla_tgt_srr_ctio *sc;
- struct qla_tgt *tgt = vha->vha_tgt.qla_tgt;
- struct qla_tgt_srr_imm *imm;
-
- tgt->ctio_srr_id++;
- cmd->cmd_flags |= BIT_15;
-
- ql_dbg(ql_dbg_tgt_mgt, vha, 0xf019,
- "qla_target(%d): CTIO with SRR status received\n", vha->vp_idx);
-
- if (!ctio) {
- ql_dbg(ql_dbg_tgt_mgt, vha, 0xf055,
- "qla_target(%d): SRR CTIO, but ctio is NULL\n",
- vha->vp_idx);
- return -EINVAL;
- }
-
- sc = kzalloc(sizeof(*sc), GFP_ATOMIC);
- if (sc != NULL) {
- sc->cmd = cmd;
- /* IRQ is already OFF */
- spin_lock(&tgt->srr_lock);
- sc->srr_id = tgt->ctio_srr_id;
- list_add_tail(&sc->srr_list_entry,
- &tgt->srr_ctio_list);
- ql_dbg(ql_dbg_tgt_mgt, vha, 0xf01a,
- "CTIO SRR %p added (id %d)\n", sc, sc->srr_id);
- if (tgt->imm_srr_id == tgt->ctio_srr_id) {
- int found = 0;
- list_for_each_entry(imm, &tgt->srr_imm_list,
- srr_list_entry) {
- if (imm->srr_id == sc->srr_id) {
- found = 1;
- break;
- }
- }
- if (found) {
- ql_dbg(ql_dbg_tgt_mgt, vha, 0xf01b,
- "Scheduling srr work\n");
- schedule_work(&tgt->srr_work);
- } else {
- ql_dbg(ql_dbg_tgt_mgt, vha, 0xf056,
- "qla_target(%d): imm_srr_id "
- "== ctio_srr_id (%d), but there is no "
- "corresponding SRR IMM, deleting CTIO "
- "SRR %p\n", vha->vp_idx,
- tgt->ctio_srr_id, sc);
- list_del(&sc->srr_list_entry);
- spin_unlock(&tgt->srr_lock);
-
- kfree(sc);
- return -EINVAL;
- }
- }
- spin_unlock(&tgt->srr_lock);
- } else {
- struct qla_tgt_srr_imm *ti;
-
- ql_dbg(ql_dbg_tgt_mgt, vha, 0xf057,
- "qla_target(%d): Unable to allocate SRR CTIO entry\n",
- vha->vp_idx);
- spin_lock(&tgt->srr_lock);
- list_for_each_entry_safe(imm, ti, &tgt->srr_imm_list,
- srr_list_entry) {
- if (imm->srr_id == tgt->ctio_srr_id) {
- ql_dbg(ql_dbg_tgt_mgt, vha, 0xf01c,
- "IMM SRR %p deleted (id %d)\n",
- imm, imm->srr_id);
- list_del(&imm->srr_list_entry);
- qlt_reject_free_srr_imm(vha, imm, 1);
- }
- }
- spin_unlock(&tgt->srr_lock);
-
- return -ENOMEM;
- }
-
- return 0;
-}
-
/*
* ha->hardware_lock supposed to be held on entry. Might drop it, then reaquire
*/
dump_stack();
}
- cmd->cmd_flags |= BIT_17;
+ cmd->trc_flags |= TRC_FLUSH;
ha->tgt.tgt_ops->free_cmd(cmd);
}
*/
cmd->sess->logout_on_delete = 0;
cmd->sess->send_els_logo = 1;
- qlt_schedule_sess_for_deletion(cmd->sess, true);
+ ql_dbg(ql_dbg_disc, vha, 0xffff,
+ "%s %d %8phC post del sess\n",
+ __func__, __LINE__, cmd->sess->port_name);
+
+ qlt_schedule_sess_for_deletion_lock(cmd->sess);
}
break;
}
- case CTIO_SRR_RECEIVED:
- ql_dbg(ql_dbg_tgt_mgt, vha, 0xf05a,
- "qla_target(%d): CTIO with SRR_RECEIVED"
- " status %x received (state %x, se_cmd %p)\n",
- vha->vp_idx, status, cmd->state, se_cmd);
- if (qlt_prepare_srr_ctio(vha, cmd, ctio) != 0)
- break;
- else
- return;
-
case CTIO_DIF_ERROR: {
struct ctio_crc_from_fw *crc =
(struct ctio_crc_from_fw *)ctio;
*/
if ((cmd->state != QLA_TGT_STATE_NEED_DATA) &&
(!cmd->aborted)) {
- cmd->cmd_flags |= BIT_13;
+ cmd->trc_flags |= TRC_CTIO_ERR;
if (qlt_term_ctio_exchange(vha, ctio, cmd, status))
return;
}
skip_term:
if (cmd->state == QLA_TGT_STATE_PROCESSED) {
- cmd->cmd_flags |= BIT_12;
+ cmd->trc_flags |= TRC_CTIO_DONE;
} else if (cmd->state == QLA_TGT_STATE_NEED_DATA) {
cmd->state = QLA_TGT_STATE_DATA_IN;
ha->tgt.tgt_ops->handle_data(cmd);
return;
} else if (cmd->aborted) {
- cmd->cmd_flags |= BIT_18;
+ cmd->trc_flags |= TRC_CTIO_ABORTED;
ql_dbg(ql_dbg_tgt_mgt, vha, 0xf01e,
"Aborted command %p (tag %lld) finished\n", cmd, se_cmd->tag);
} else {
- cmd->cmd_flags |= BIT_19;
+ cmd->trc_flags |= TRC_CTIO_STRANGE;
ql_dbg(ql_dbg_tgt_mgt, vha, 0xf05c,
"qla_target(%d): A command in state (%d) should "
"not return a CTIO complete\n", vha->vp_idx, cmd->state);
return fcp_task_attr;
}
-static struct qla_tgt_sess *qlt_make_local_sess(struct scsi_qla_host *,
+static struct fc_port *qlt_make_local_sess(struct scsi_qla_host *,
uint8_t *);
/*
* Process context for I/O path into tcm_qla2xxx code
scsi_qla_host_t *vha = cmd->vha;
struct qla_hw_data *ha = vha->hw;
struct qla_tgt *tgt = vha->vha_tgt.qla_tgt;
- struct qla_tgt_sess *sess = cmd->sess;
+ struct fc_port *sess = cmd->sess;
struct atio_from_isp *atio = &cmd->atio;
unsigned char *cdb;
unsigned long flags;
int ret, fcp_task_attr, data_dir, bidi = 0;
cmd->cmd_in_wq = 0;
- cmd->cmd_flags |= BIT_1;
+ cmd->trc_flags |= TRC_DO_WORK;
if (tgt->tgt_stop)
goto out_term;
* Drop extra session reference from qla_tgt_handle_cmd_for_atio*(
*/
spin_lock_irqsave(&ha->tgt.sess_lock, flags);
- qlt_put_sess(sess);
+ ha->tgt.tgt_ops->put_sess(sess);
spin_unlock_irqrestore(&ha->tgt.sess_lock, flags);
return;
* cmd has not sent to target yet, so pass NULL as the second
* argument to qlt_send_term_exchange() and free the memory here.
*/
- cmd->cmd_flags |= BIT_2;
+ cmd->trc_flags |= TRC_DO_WORK_ERR;
spin_lock_irqsave(&ha->hardware_lock, flags);
qlt_send_term_exchange(vha, NULL, &cmd->atio, 1, 0);
spin_unlock_irqrestore(&ha->hardware_lock, flags);
spin_lock_irqsave(&ha->tgt.sess_lock, flags);
- qlt_put_sess(sess);
+ ha->tgt.tgt_ops->put_sess(sess);
spin_unlock_irqrestore(&ha->tgt.sess_lock, flags);
}
}
static struct qla_tgt_cmd *qlt_get_tag(scsi_qla_host_t *vha,
- struct qla_tgt_sess *sess,
+ struct fc_port *sess,
struct atio_from_isp *atio)
{
struct se_session *se_sess = sess->se_sess;
cmd->loop_id = sess->loop_id;
cmd->conf_compl_supported = sess->conf_compl_supported;
- cmd->cmd_flags = 0;
+ cmd->trc_flags = 0;
cmd->jiffies_at_alloc = get_jiffies_64();
cmd->reset_count = vha->hw->chip_reset;
struct qla_tgt_sess_op, work);
scsi_qla_host_t *vha = op->vha;
struct qla_hw_data *ha = vha->hw;
- struct qla_tgt_sess *sess;
+ struct fc_port *sess;
struct qla_tgt_cmd *cmd;
unsigned long flags;
uint8_t *s_id = op->atio.u.isp24.fcp_hdr.s_id;
if (!cmd) {
spin_lock_irqsave(&ha->hardware_lock, flags);
qlt_send_busy(vha, &op->atio, SAM_STAT_BUSY);
- qlt_put_sess(sess);
+ ha->tgt.tgt_ops->put_sess(sess);
spin_unlock_irqrestore(&ha->hardware_lock, flags);
kfree(op);
return;
}
+
/*
* __qlt_do_work() will call qlt_put_sess() to release
* the extra reference taken above by qlt_make_local_sess()
__qlt_do_work(cmd);
kfree(op);
return;
-
out_term:
spin_lock_irqsave(&ha->hardware_lock, flags);
qlt_send_term_exchange(vha, NULL, &op->atio, 1, 0);
spin_unlock_irqrestore(&ha->hardware_lock, flags);
kfree(op);
-
}
/* ha->hardware_lock supposed to be held on entry */
{
struct qla_hw_data *ha = vha->hw;
struct qla_tgt *tgt = vha->vha_tgt.qla_tgt;
- struct qla_tgt_sess *sess;
+ struct fc_port *sess;
struct qla_tgt_cmd *cmd;
+ unsigned long flags;
if (unlikely(tgt->tgt_stop)) {
ql_dbg(ql_dbg_io, vha, 0x3061,
/* Another WWN used to have our s_id. Our PLOGI scheduled its
* session deletion, but it's still in sess_del_work wq */
- if (sess->deleted == QLA_SESS_DELETION_IN_PROGRESS) {
+ if (sess->deleted) {
ql_dbg(ql_dbg_io, vha, 0x3061,
"New command while old session %p is being deleted\n",
sess);
/*
* Do kref_get() before returning + dropping qla_hw_data->hardware_lock.
*/
- kref_get(&sess->sess_kref);
+ if (!kref_get_unless_zero(&sess->sess_kref)) {
+ ql_dbg(ql_dbg_tgt, vha, 0xffff,
+ "%s: kref_get fail, %8phC oxid %x \n",
+ __func__, sess->port_name,
+ be16_to_cpu(atio->u.isp24.fcp_hdr.ox_id));
+ return -EFAULT;
+ }
cmd = qlt_get_tag(vha, sess, atio);
if (!cmd) {
ql_dbg(ql_dbg_io, vha, 0x3062,
"qla_target(%d): Allocation of cmd failed\n", vha->vp_idx);
- qlt_put_sess(sess);
+ spin_lock_irqsave(&ha->tgt.sess_lock, flags);
+ ha->tgt.tgt_ops->put_sess(sess);
+ spin_unlock_irqrestore(&ha->tgt.sess_lock, flags);
return -ENOMEM;
}
cmd->cmd_in_wq = 1;
- cmd->cmd_flags |= BIT_0;
+ cmd->trc_flags |= TRC_NEW_CMD;
cmd->se_cmd.cpuid = ha->msix_count ?
ha->tgt.rspq_vector_cpuid : WORK_CPU_UNBOUND;
- spin_lock(&vha->cmd_list_lock);
+ spin_lock_irqsave(&vha->cmd_list_lock, flags);
list_add_tail(&cmd->cmd_list, &vha->qla_cmd_list);
- spin_unlock(&vha->cmd_list_lock);
+ spin_unlock_irqrestore(&vha->cmd_list_lock, flags);
INIT_WORK(&cmd->work, qlt_do_work);
if (ha->msix_count) {
}
/* ha->hardware_lock supposed to be held on entry */
-static int qlt_issue_task_mgmt(struct qla_tgt_sess *sess, uint32_t lun,
+static int qlt_issue_task_mgmt(struct fc_port *sess, u64 lun,
int fn, void *iocb, int flags)
{
struct scsi_qla_host *vha = sess->vha;
struct qla_tgt_mgmt_cmd *mcmd;
struct atio_from_isp *a = (struct atio_from_isp *)iocb;
int res;
- uint8_t tmr_func;
mcmd = mempool_alloc(qla_tgt_mgmt_cmd_mempool, GFP_ATOMIC);
if (!mcmd) {
mcmd->reset_count = vha->hw->chip_reset;
switch (fn) {
- case QLA_TGT_CLEAR_ACA:
- ql_dbg(ql_dbg_tgt_tmr, vha, 0x10000,
- "qla_target(%d): CLEAR_ACA received\n", sess->vha->vp_idx);
- tmr_func = TMR_CLEAR_ACA;
- break;
-
- case QLA_TGT_TARGET_RESET:
- ql_dbg(ql_dbg_tgt_tmr, vha, 0x10001,
- "qla_target(%d): TARGET_RESET received\n",
- sess->vha->vp_idx);
- tmr_func = TMR_TARGET_WARM_RESET;
- break;
-
case QLA_TGT_LUN_RESET:
- ql_dbg(ql_dbg_tgt_tmr, vha, 0x10002,
- "qla_target(%d): LUN_RESET received\n", sess->vha->vp_idx);
- tmr_func = TMR_LUN_RESET;
- abort_cmds_for_lun(vha, lun, a->u.isp24.fcp_hdr.s_id);
- break;
-
- case QLA_TGT_CLEAR_TS:
- ql_dbg(ql_dbg_tgt_tmr, vha, 0x10003,
- "qla_target(%d): CLEAR_TS received\n", sess->vha->vp_idx);
- tmr_func = TMR_CLEAR_TASK_SET;
- break;
-
- case QLA_TGT_ABORT_TS:
- ql_dbg(ql_dbg_tgt_tmr, vha, 0x10004,
- "qla_target(%d): ABORT_TS received\n", sess->vha->vp_idx);
- tmr_func = TMR_ABORT_TASK_SET;
- break;
-#if 0
- case QLA_TGT_ABORT_ALL:
- ql_dbg(ql_dbg_tgt_tmr, vha, 0x10005,
- "qla_target(%d): Doing ABORT_ALL_TASKS\n",
- sess->vha->vp_idx);
- tmr_func = 0;
- break;
-
- case QLA_TGT_ABORT_ALL_SESS:
- ql_dbg(ql_dbg_tgt_tmr, vha, 0x10006,
- "qla_target(%d): Doing ABORT_ALL_TASKS_SESS\n",
- sess->vha->vp_idx);
- tmr_func = 0;
- break;
-
- case QLA_TGT_NEXUS_LOSS_SESS:
- ql_dbg(ql_dbg_tgt_tmr, vha, 0x10007,
- "qla_target(%d): Doing NEXUS_LOSS_SESS\n",
- sess->vha->vp_idx);
- tmr_func = 0;
- break;
-
- case QLA_TGT_NEXUS_LOSS:
- ql_dbg(ql_dbg_tgt_tmr, vha, 0x10008,
- "qla_target(%d): Doing NEXUS_LOSS\n", sess->vha->vp_idx);
- tmr_func = 0;
- break;
-#endif
- default:
- ql_dbg(ql_dbg_tgt_tmr, vha, 0x1000a,
- "qla_target(%d): Unknown task mgmt fn 0x%x\n",
- sess->vha->vp_idx, fn);
- mempool_free(mcmd, qla_tgt_mgmt_cmd_mempool);
- return -ENOSYS;
+ abort_cmds_for_lun(vha, lun, a->u.isp24.fcp_hdr.s_id);
+ break;
}
- res = ha->tgt.tgt_ops->handle_tmr(mcmd, lun, tmr_func, 0);
+ res = ha->tgt.tgt_ops->handle_tmr(mcmd, lun, mcmd->tmr_func, 0);
if (res != 0) {
ql_dbg(ql_dbg_tgt_tmr, vha, 0x1000b,
"qla_target(%d): tgt.tgt_ops->handle_tmr() failed: %d\n",
struct atio_from_isp *a = (struct atio_from_isp *)iocb;
struct qla_hw_data *ha = vha->hw;
struct qla_tgt *tgt;
- struct qla_tgt_sess *sess;
+ struct fc_port *sess;
uint32_t lun, unpacked_lun;
int fn;
unsigned long flags;
sizeof(struct atio_from_isp));
}
- if (sess->deleted == QLA_SESS_DELETION_IN_PROGRESS)
+ if (sess->deleted)
return -EFAULT;
return qlt_issue_task_mgmt(sess, unpacked_lun, fn, iocb, 0);
/* ha->hardware_lock supposed to be held on entry */
static int __qlt_abort_task(struct scsi_qla_host *vha,
- struct imm_ntfy_from_isp *iocb, struct qla_tgt_sess *sess)
+ struct imm_ntfy_from_isp *iocb, struct fc_port *sess)
{
struct atio_from_isp *a = (struct atio_from_isp *)iocb;
struct qla_hw_data *ha = vha->hw;
lun = a->u.isp24.fcp_cmnd.lun;
unpacked_lun = scsilun_to_int((struct scsi_lun *)&lun);
mcmd->reset_count = vha->hw->chip_reset;
+ mcmd->tmr_func = QLA_TGT_2G_ABORT_TASK;
- rc = ha->tgt.tgt_ops->handle_tmr(mcmd, unpacked_lun, TMR_ABORT_TASK,
+ rc = ha->tgt.tgt_ops->handle_tmr(mcmd, unpacked_lun, mcmd->tmr_func,
le16_to_cpu(iocb->u.isp2x.seq_id));
if (rc != 0) {
ql_dbg(ql_dbg_tgt_mgt, vha, 0xf060,
struct imm_ntfy_from_isp *iocb)
{
struct qla_hw_data *ha = vha->hw;
- struct qla_tgt_sess *sess;
+ struct fc_port *sess;
int loop_id;
unsigned long flags;
void qlt_logo_completion_handler(fc_port_t *fcport, int rc)
{
- if (fcport->tgt_session) {
- if (rc != MBS_COMMAND_COMPLETE) {
- ql_dbg(ql_dbg_tgt_mgt, fcport->vha, 0xf093,
- "%s: se_sess %p / sess %p from"
- " port %8phC loop_id %#04x s_id %02x:%02x:%02x"
- " LOGO failed: %#x\n",
- __func__,
- fcport->tgt_session->se_sess,
- fcport->tgt_session,
- fcport->port_name, fcport->loop_id,
- fcport->d_id.b.domain, fcport->d_id.b.area,
- fcport->d_id.b.al_pa, rc);
- }
-
- fcport->tgt_session->logout_completed = 1;
+ if (rc != MBS_COMMAND_COMPLETE) {
+ ql_dbg(ql_dbg_tgt_mgt, fcport->vha, 0xf093,
+ "%s: se_sess %p / sess %p from"
+ " port %8phC loop_id %#04x s_id %02x:%02x:%02x"
+ " LOGO failed: %#x\n",
+ __func__,
+ fcport->se_sess,
+ fcport,
+ fcport->port_name, fcport->loop_id,
+ fcport->d_id.b.domain, fcport->d_id.b.area,
+ fcport->d_id.b.al_pa, rc);
}
+
+ fcport->logout_completed = 1;
}
/*
* deletion. Returns existing session with matching wwn if present.
* Null otherwise.
*/
-static struct qla_tgt_sess *
-qlt_find_sess_invalidate_other(struct qla_tgt *tgt, uint64_t wwn,
- port_id_t port_id, uint16_t loop_id, struct qla_tgt_sess **conflict_sess)
+struct fc_port *
+qlt_find_sess_invalidate_other(scsi_qla_host_t *vha, uint64_t wwn,
+ port_id_t port_id, uint16_t loop_id, struct fc_port **conflict_sess)
{
- struct qla_tgt_sess *sess = NULL, *other_sess;
+ struct fc_port *sess = NULL, *other_sess;
uint64_t other_wwn;
*conflict_sess = NULL;
- list_for_each_entry(other_sess, &tgt->sess_list, sess_list_entry) {
+ list_for_each_entry(other_sess, &vha->vp_fcports, list) {
other_wwn = wwn_to_u64(other_sess->port_name);
}
/* find other sess with nport_id collision */
- if (port_id.b24 == other_sess->s_id.b24) {
+ if (port_id.b24 == other_sess->d_id.b24) {
if (loop_id != other_sess->loop_id) {
- ql_dbg(ql_dbg_tgt_tmr, tgt->vha, 0x1000c,
+ ql_dbg(ql_dbg_tgt_tmr, vha, 0x1000c,
"Invalidating sess %p loop_id %d wwn %llx.\n",
other_sess, other_sess->loop_id, other_wwn);
* Another wwn used to have our s_id/loop_id
* kill the session, but don't free the loop_id
*/
+ ql_dbg(ql_dbg_tgt_tmr, vha, 0xffff,
+ "Invalidating sess %p loop_id %d wwn %llx.\n",
+ other_sess, other_sess->loop_id, other_wwn);
+
+
other_sess->keep_nport_handle = 1;
*conflict_sess = other_sess;
qlt_schedule_sess_for_deletion(other_sess,
}
/* find other sess with nport handle collision */
- if (loop_id == other_sess->loop_id) {
- ql_dbg(ql_dbg_tgt_tmr, tgt->vha, 0x1000d,
+ if ((loop_id == other_sess->loop_id) &&
+ (loop_id != FC_NO_LOOP_ID)) {
+ ql_dbg(ql_dbg_tgt_tmr, vha, 0x1000d,
"Invalidating sess %p loop_id %d wwn %llx.\n",
other_sess, other_sess->loop_id, other_wwn);
spin_lock(&vha->cmd_list_lock);
list_for_each_entry(op, &vha->qla_sess_op_cmd_list, cmd_list) {
uint32_t op_key = sid_to_key(op->atio.u.isp24.fcp_hdr.s_id);
+
+ if (op_key == key) {
+ op->aborted = true;
+ count++;
+ }
+ }
+
+ list_for_each_entry(op, &vha->unknown_atio_list, cmd_list) {
+ uint32_t op_key = sid_to_key(op->atio.u.isp24.fcp_hdr.s_id);
if (op_key == key) {
op->aborted = true;
count++;
}
}
+
list_for_each_entry(cmd, &vha->qla_cmd_list, cmd_list) {
uint32_t cmd_key = sid_to_key(cmd->atio.u.isp24.fcp_hdr.s_id);
if (cmd_key == key) {
{
struct qla_tgt *tgt = vha->vha_tgt.qla_tgt;
struct qla_hw_data *ha = vha->hw;
- struct qla_tgt_sess *sess = NULL, *conflict_sess = NULL;
+ struct fc_port *sess = NULL, *conflict_sess = NULL;
uint64_t wwn;
port_id_t port_id;
uint16_t loop_id;
uint16_t wd3_lo;
int res = 0;
- qlt_plogi_ack_t *pla;
+ struct qlt_plogi_ack_t *pla;
unsigned long flags;
wwn = wwn_to_u64(iocb->u.isp24.port_name);
loop_id = le16_to_cpu(iocb->u.isp24.nport_handle);
- ql_dbg(ql_dbg_tgt_mgt, vha, 0xf026,
- "qla_target(%d): Port ID: 0x%3phC ELS opcode: 0x%02x\n",
- vha->vp_idx, iocb->u.isp24.port_id, iocb->u.isp24.status_subcode);
+ ql_dbg(ql_dbg_disc, vha, 0xf026,
+ "qla_target(%d): Port ID: %02x:%02x:%02x ELS opcode: 0x%02x lid %d %8phC\n",
+ vha->vp_idx, iocb->u.isp24.port_id[2],
+ iocb->u.isp24.port_id[1], iocb->u.isp24.port_id[0],
+ iocb->u.isp24.status_subcode, loop_id,
+ iocb->u.isp24.port_name);
/* res = 1 means ack at the end of thread
* res = 0 means ack async/later.
if (wwn) {
spin_lock_irqsave(&tgt->ha->tgt.sess_lock, flags);
- sess = qlt_find_sess_invalidate_other(tgt, wwn,
- port_id, loop_id, &conflict_sess);
+ sess = qlt_find_sess_invalidate_other(vha, wwn,
+ port_id, loop_id, &conflict_sess);
spin_unlock_irqrestore(&tgt->ha->tgt.sess_lock, flags);
}
- if (IS_SW_RESV_ADDR(port_id) || (!sess && !conflict_sess)) {
+ if (IS_SW_RESV_ADDR(port_id)) {
res = 1;
break;
}
pla = qlt_plogi_ack_find_add(vha, &port_id, iocb);
if (!pla) {
qlt_send_term_imm_notif(vha, iocb, 1);
+ break;
+ }
+
+ res = 0;
+
+ if (conflict_sess) {
+ conflict_sess->login_gen++;
+ qlt_plogi_ack_link(vha, pla, conflict_sess,
+ QLT_PLOGI_LINK_CONFLICT);
+ }
+
+ if (!sess) {
+ pla->ref_count++;
+ qla24xx_post_newsess_work(vha, &port_id,
+ iocb->u.isp24.port_name, pla);
+ res = 0;
+ break;
+ }
+
+ qlt_plogi_ack_link(vha, pla, sess, QLT_PLOGI_LINK_SAME_WWN);
+ sess->fw_login_state = DSC_LS_PLOGI_PEND;
+ sess->d_id = port_id;
+ sess->login_gen++;
- res = 0;
+ switch (sess->disc_state) {
+ case DSC_DELETED:
+ qlt_plogi_ack_unref(vha, pla);
break;
- }
- res = 0;
+ default:
+ /*
+ * Under normal circumstances we want to release nport handle
+ * during LOGO process to avoid nport handle leaks inside FW.
+ * The exception is when LOGO is done while another PLOGI with
+ * the same nport handle is waiting as might be the case here.
+ * Note: there is always a possibily of a race where session
+ * deletion has already started for other reasons (e.g. ACL
+ * removal) and now PLOGI arrives:
+ * 1. if PLOGI arrived in FW after nport handle has been freed,
+ * FW must have assigned this PLOGI a new/same handle and we
+ * can proceed ACK'ing it as usual when session deletion
+ * completes.
+ * 2. if PLOGI arrived in FW before LOGO with LCF_FREE_NPORT
+ * bit reached it, the handle has now been released. We'll
+ * get an error when we ACK this PLOGI. Nothing will be sent
+ * back to initiator. Initiator should eventually retry
+ * PLOGI and situation will correct itself.
+ */
+ sess->keep_nport_handle = ((sess->loop_id == loop_id) &&
+ (sess->d_id.b24 == port_id.b24));
- if (conflict_sess)
- qlt_plogi_ack_link(vha, pla, conflict_sess,
- QLT_PLOGI_LINK_CONFLICT);
+ ql_dbg(ql_dbg_disc, vha, 0xffff,
+ "%s %d %8phC post del sess\n",
+ __func__, __LINE__, sess->port_name);
- if (!sess)
+
+ qlt_schedule_sess_for_deletion_lock(sess);
break;
+ }
- qlt_plogi_ack_link(vha, pla, sess, QLT_PLOGI_LINK_SAME_WWN);
- /*
- * Under normal circumstances we want to release nport handle
- * during LOGO process to avoid nport handle leaks inside FW.
- * The exception is when LOGO is done while another PLOGI with
- * the same nport handle is waiting as might be the case here.
- * Note: there is always a possibily of a race where session
- * deletion has already started for other reasons (e.g. ACL
- * removal) and now PLOGI arrives:
- * 1. if PLOGI arrived in FW after nport handle has been freed,
- * FW must have assigned this PLOGI a new/same handle and we
- * can proceed ACK'ing it as usual when session deletion
- * completes.
- * 2. if PLOGI arrived in FW before LOGO with LCF_FREE_NPORT
- * bit reached it, the handle has now been released. We'll
- * get an error when we ACK this PLOGI. Nothing will be sent
- * back to initiator. Initiator should eventually retry
- * PLOGI and situation will correct itself.
- */
- sess->keep_nport_handle = ((sess->loop_id == loop_id) &&
- (sess->s_id.b24 == port_id.b24));
- qlt_schedule_sess_for_deletion(sess, true);
break;
case ELS_PRLI:
if (wwn) {
spin_lock_irqsave(&tgt->ha->tgt.sess_lock, flags);
- sess = qlt_find_sess_invalidate_other(tgt, wwn, port_id,
- loop_id, &conflict_sess);
+ sess = qlt_find_sess_invalidate_other(vha, wwn, port_id,
+ loop_id, &conflict_sess);
spin_unlock_irqrestore(&tgt->ha->tgt.sess_lock, flags);
}
}
if (sess != NULL) {
- if (sess->deleted) {
+ if (sess->fw_login_state == DSC_LS_PLOGI_PEND) {
/*
* Impatient initiator sent PRLI before last
* PLOGI could finish. Will force him to re-try,
sess->local = 0;
sess->loop_id = loop_id;
- sess->s_id = port_id;
+ sess->d_id = port_id;
+ sess->fw_login_state = DSC_LS_PRLI_PEND;
if (wd3_lo & BIT_7)
sess->conf_compl_supported = 1;
+ if ((wd3_lo & BIT_4) == 0)
+ sess->port_type = FCT_INITIATOR;
+ else
+ sess->port_type = FCT_TARGET;
}
res = 1; /* send notify ack */
set_bit(LOCAL_LOOP_UPDATE, &vha->dpc_flags);
qla2xxx_wake_dpc(vha);
} else {
- /* todo: else - create sess here. */
- res = 1; /* send notify ack */
- }
+ if (sess) {
+ ql_dbg(ql_dbg_disc, vha, 0xffff,
+ "%s %d %8phC post nack\n",
+ __func__, __LINE__, sess->port_name);
+ qla24xx_post_nack_work(vha, sess, iocb,
+ SRB_NACK_PRLI);
+ res = 0;
+ }
+ }
break;
+
+ case ELS_TPRLO:
+ if (le16_to_cpu(iocb->u.isp24.flags) &
+ NOTIFY24XX_FLAGS_GLOBAL_TPRLO) {
+ loop_id = 0xFFFF;
+ qlt_reset(vha, iocb, QLA_TGT_NEXUS_LOSS);
+ res = 1;
+ break;
+ }
+ /* drop through */
case ELS_LOGO:
case ELS_PRLO:
+ spin_lock_irqsave(&ha->tgt.sess_lock, flags);
+ sess = qla2x00_find_fcport_by_loopid(vha, loop_id);
+ spin_unlock_irqrestore(&ha->tgt.sess_lock, flags);
+
+ if (sess) {
+ sess->login_gen++;
+ sess->fw_login_state = DSC_LS_LOGO_PEND;
+ sess->logo_ack_needed = 1;
+ memcpy(sess->iocb, iocb, IOCB_SIZE);
+ }
+
res = qlt_reset(vha, iocb, QLA_TGT_NEXUS_LOSS_SESS);
+
+ ql_dbg(ql_dbg_disc, vha, 0xffff,
+ "%s: logo %llx res %d sess %p ",
+ __func__, wwn, res, sess);
+ if (res == 0) {
+ /*
+ * cmd went upper layer, look for qlt_xmit_tm_rsp()
+ * for LOGO_ACK & sess delete
+ */
+ BUG_ON(!sess);
+ res = 0;
+ } else {
+ /* cmd did not go to upper layer. */
+ if (sess) {
+ qlt_schedule_sess_for_deletion_lock(sess);
+ res = 0;
+ }
+ /* else logo will be ack */
+ }
break;
case ELS_PDISC:
case ELS_ADISC:
0, 0, 0, 0, 0, 0);
tgt->link_reinit_iocb_pending = 0;
}
+
+ sess = qla2x00_find_fcport_by_wwpn(vha,
+ iocb->u.isp24.port_name, 1);
+ if (sess) {
+ ql_dbg(ql_dbg_disc, vha, 0xffff,
+ "sess %p lid %d|%d DS %d LS %d\n",
+ sess, sess->loop_id, loop_id,
+ sess->disc_state, sess->fw_login_state);
+ }
+
res = 1; /* send notify ack */
break;
}
return res;
}
-static int qlt_set_data_offset(struct qla_tgt_cmd *cmd, uint32_t offset)
-{
-#if 1
- /*
- * FIXME: Reject non zero SRR relative offset until we can test
- * this code properly.
- */
- pr_debug("Rejecting non zero SRR rel_offs: %u\n", offset);
- return -1;
-#else
- struct scatterlist *sg, *sgp, *sg_srr, *sg_srr_start = NULL;
- size_t first_offset = 0, rem_offset = offset, tmp = 0;
- int i, sg_srr_cnt, bufflen = 0;
-
- ql_dbg(ql_dbg_tgt, cmd->vha, 0xe023,
- "Entering qla_tgt_set_data_offset: cmd: %p, cmd->sg: %p, "
- "cmd->sg_cnt: %u, direction: %d\n",
- cmd, cmd->sg, cmd->sg_cnt, cmd->dma_data_direction);
-
- if (!cmd->sg || !cmd->sg_cnt) {
- ql_dbg(ql_dbg_tgt, cmd->vha, 0xe055,
- "Missing cmd->sg or zero cmd->sg_cnt in"
- " qla_tgt_set_data_offset\n");
- return -EINVAL;
- }
- /*
- * Walk the current cmd->sg list until we locate the new sg_srr_start
- */
- for_each_sg(cmd->sg, sg, cmd->sg_cnt, i) {
- ql_dbg(ql_dbg_tgt, cmd->vha, 0xe024,
- "sg[%d]: %p page: %p, length: %d, offset: %d\n",
- i, sg, sg_page(sg), sg->length, sg->offset);
-
- if ((sg->length + tmp) > offset) {
- first_offset = rem_offset;
- sg_srr_start = sg;
- ql_dbg(ql_dbg_tgt, cmd->vha, 0xe025,
- "Found matching sg[%d], using %p as sg_srr_start, "
- "and using first_offset: %zu\n", i, sg,
- first_offset);
- break;
- }
- tmp += sg->length;
- rem_offset -= sg->length;
- }
-
- if (!sg_srr_start) {
- ql_dbg(ql_dbg_tgt, cmd->vha, 0xe056,
- "Unable to locate sg_srr_start for offset: %u\n", offset);
- return -EINVAL;
- }
- sg_srr_cnt = (cmd->sg_cnt - i);
-
- sg_srr = kzalloc(sizeof(struct scatterlist) * sg_srr_cnt, GFP_KERNEL);
- if (!sg_srr) {
- ql_dbg(ql_dbg_tgt, cmd->vha, 0xe057,
- "Unable to allocate sgp\n");
- return -ENOMEM;
- }
- sg_init_table(sg_srr, sg_srr_cnt);
- sgp = &sg_srr[0];
- /*
- * Walk the remaining list for sg_srr_start, mapping to the newly
- * allocated sg_srr taking first_offset into account.
- */
- for_each_sg(sg_srr_start, sg, sg_srr_cnt, i) {
- if (first_offset) {
- sg_set_page(sgp, sg_page(sg),
- (sg->length - first_offset), first_offset);
- first_offset = 0;
- } else {
- sg_set_page(sgp, sg_page(sg), sg->length, 0);
- }
- bufflen += sgp->length;
-
- sgp = sg_next(sgp);
- if (!sgp)
- break;
- }
-
- cmd->sg = sg_srr;
- cmd->sg_cnt = sg_srr_cnt;
- cmd->bufflen = bufflen;
- cmd->offset += offset;
- cmd->free_sg = 1;
-
- ql_dbg(ql_dbg_tgt, cmd->vha, 0xe026, "New cmd->sg: %p\n", cmd->sg);
- ql_dbg(ql_dbg_tgt, cmd->vha, 0xe027, "New cmd->sg_cnt: %u\n",
- cmd->sg_cnt);
- ql_dbg(ql_dbg_tgt, cmd->vha, 0xe028, "New cmd->bufflen: %u\n",
- cmd->bufflen);
- ql_dbg(ql_dbg_tgt, cmd->vha, 0xe029, "New cmd->offset: %u\n",
- cmd->offset);
-
- if (cmd->sg_cnt < 0)
- BUG();
-
- if (cmd->bufflen < 0)
- BUG();
-
- return 0;
-#endif
-}
-
-static inline int qlt_srr_adjust_data(struct qla_tgt_cmd *cmd,
- uint32_t srr_rel_offs, int *xmit_type)
-{
- int res = 0, rel_offs;
-
- rel_offs = srr_rel_offs - cmd->offset;
- ql_dbg(ql_dbg_tgt_mgt, cmd->vha, 0xf027, "srr_rel_offs=%d, rel_offs=%d",
- srr_rel_offs, rel_offs);
-
- *xmit_type = QLA_TGT_XMIT_ALL;
-
- if (rel_offs < 0) {
- ql_dbg(ql_dbg_tgt_mgt, cmd->vha, 0xf062,
- "qla_target(%d): SRR rel_offs (%d) < 0",
- cmd->vha->vp_idx, rel_offs);
- res = -1;
- } else if (rel_offs == cmd->bufflen)
- *xmit_type = QLA_TGT_XMIT_STATUS;
- else if (rel_offs > 0)
- res = qlt_set_data_offset(cmd, rel_offs);
-
- return res;
-}
-
-/* No locks, thread context */
-static void qlt_handle_srr(struct scsi_qla_host *vha,
- struct qla_tgt_srr_ctio *sctio, struct qla_tgt_srr_imm *imm)
-{
- struct imm_ntfy_from_isp *ntfy =
- (struct imm_ntfy_from_isp *)&imm->imm_ntfy;
- struct qla_hw_data *ha = vha->hw;
- struct qla_tgt_cmd *cmd = sctio->cmd;
- struct se_cmd *se_cmd = &cmd->se_cmd;
- unsigned long flags;
- int xmit_type = 0, resp = 0;
- uint32_t offset;
- uint16_t srr_ui;
-
- offset = le32_to_cpu(ntfy->u.isp24.srr_rel_offs);
- srr_ui = ntfy->u.isp24.srr_ui;
-
- ql_dbg(ql_dbg_tgt_mgt, vha, 0xf028, "SRR cmd %p, srr_ui %x\n",
- cmd, srr_ui);
-
- switch (srr_ui) {
- case SRR_IU_STATUS:
- spin_lock_irqsave(&ha->hardware_lock, flags);
- qlt_send_notify_ack(vha, ntfy,
- 0, 0, 0, NOTIFY_ACK_SRR_FLAGS_ACCEPT, 0, 0);
- spin_unlock_irqrestore(&ha->hardware_lock, flags);
- xmit_type = QLA_TGT_XMIT_STATUS;
- resp = 1;
- break;
- case SRR_IU_DATA_IN:
- if (!cmd->sg || !cmd->sg_cnt) {
- ql_dbg(ql_dbg_tgt_mgt, vha, 0xf063,
- "Unable to process SRR_IU_DATA_IN due to"
- " missing cmd->sg, state: %d\n", cmd->state);
- dump_stack();
- goto out_reject;
- }
- if (se_cmd->scsi_status != 0) {
- ql_dbg(ql_dbg_tgt, vha, 0xe02a,
- "Rejecting SRR_IU_DATA_IN with non GOOD "
- "scsi_status\n");
- goto out_reject;
- }
- cmd->bufflen = se_cmd->data_length;
-
- if (qlt_has_data(cmd)) {
- if (qlt_srr_adjust_data(cmd, offset, &xmit_type) != 0)
- goto out_reject;
- spin_lock_irqsave(&ha->hardware_lock, flags);
- qlt_send_notify_ack(vha, ntfy,
- 0, 0, 0, NOTIFY_ACK_SRR_FLAGS_ACCEPT, 0, 0);
- spin_unlock_irqrestore(&ha->hardware_lock, flags);
- resp = 1;
- } else {
- ql_dbg(ql_dbg_tgt_mgt, vha, 0xf064,
- "qla_target(%d): SRR for in data for cmd without them (tag %lld, SCSI status %d), reject",
- vha->vp_idx, se_cmd->tag,
- cmd->se_cmd.scsi_status);
- goto out_reject;
- }
- break;
- case SRR_IU_DATA_OUT:
- if (!cmd->sg || !cmd->sg_cnt) {
- ql_dbg(ql_dbg_tgt_mgt, vha, 0xf065,
- "Unable to process SRR_IU_DATA_OUT due to"
- " missing cmd->sg\n");
- dump_stack();
- goto out_reject;
- }
- if (se_cmd->scsi_status != 0) {
- ql_dbg(ql_dbg_tgt, vha, 0xe02b,
- "Rejecting SRR_IU_DATA_OUT"
- " with non GOOD scsi_status\n");
- goto out_reject;
- }
- cmd->bufflen = se_cmd->data_length;
-
- if (qlt_has_data(cmd)) {
- if (qlt_srr_adjust_data(cmd, offset, &xmit_type) != 0)
- goto out_reject;
- spin_lock_irqsave(&ha->hardware_lock, flags);
- qlt_send_notify_ack(vha, ntfy,
- 0, 0, 0, NOTIFY_ACK_SRR_FLAGS_ACCEPT, 0, 0);
- spin_unlock_irqrestore(&ha->hardware_lock, flags);
- if (xmit_type & QLA_TGT_XMIT_DATA) {
- cmd->cmd_flags |= BIT_8;
- qlt_rdy_to_xfer(cmd);
- }
- } else {
- ql_dbg(ql_dbg_tgt_mgt, vha, 0xf066,
- "qla_target(%d): SRR for out data for cmd without them (tag %lld, SCSI status %d), reject",
- vha->vp_idx, se_cmd->tag, cmd->se_cmd.scsi_status);
- goto out_reject;
- }
- break;
- default:
- ql_dbg(ql_dbg_tgt_mgt, vha, 0xf067,
- "qla_target(%d): Unknown srr_ui value %x",
- vha->vp_idx, srr_ui);
- goto out_reject;
- }
-
- /* Transmit response in case of status and data-in cases */
- if (resp) {
- cmd->cmd_flags |= BIT_7;
- qlt_xmit_response(cmd, xmit_type, se_cmd->scsi_status);
- }
-
- return;
-
-out_reject:
- spin_lock_irqsave(&ha->hardware_lock, flags);
- qlt_send_notify_ack(vha, ntfy, 0, 0, 0,
- NOTIFY_ACK_SRR_FLAGS_REJECT,
- NOTIFY_ACK_SRR_REJECT_REASON_UNABLE_TO_PERFORM,
- NOTIFY_ACK_SRR_FLAGS_REJECT_EXPL_NO_EXPL);
- if (cmd->state == QLA_TGT_STATE_NEED_DATA) {
- cmd->state = QLA_TGT_STATE_DATA_IN;
- dump_stack();
- } else {
- cmd->cmd_flags |= BIT_9;
- qlt_send_term_exchange(vha, cmd, &cmd->atio, 1, 0);
- }
- spin_unlock_irqrestore(&ha->hardware_lock, flags);
-}
-
-static void qlt_reject_free_srr_imm(struct scsi_qla_host *vha,
- struct qla_tgt_srr_imm *imm, int ha_locked)
-{
- struct qla_hw_data *ha = vha->hw;
- unsigned long flags = 0;
-
-#ifndef __CHECKER__
- if (!ha_locked)
- spin_lock_irqsave(&ha->hardware_lock, flags);
-#endif
-
- qlt_send_notify_ack(vha, (void *)&imm->imm_ntfy, 0, 0, 0,
- NOTIFY_ACK_SRR_FLAGS_REJECT,
- NOTIFY_ACK_SRR_REJECT_REASON_UNABLE_TO_PERFORM,
- NOTIFY_ACK_SRR_FLAGS_REJECT_EXPL_NO_EXPL);
-
-#ifndef __CHECKER__
- if (!ha_locked)
- spin_unlock_irqrestore(&ha->hardware_lock, flags);
-#endif
-
- kfree(imm);
-}
-
-static void qlt_handle_srr_work(struct work_struct *work)
-{
- struct qla_tgt *tgt = container_of(work, struct qla_tgt, srr_work);
- struct scsi_qla_host *vha = tgt->vha;
- struct qla_tgt_srr_ctio *sctio;
- unsigned long flags;
-
- ql_dbg(ql_dbg_tgt_mgt, vha, 0xf029, "Entering SRR work (tgt %p)\n",
- tgt);
-
-restart:
- spin_lock_irqsave(&tgt->srr_lock, flags);
- list_for_each_entry(sctio, &tgt->srr_ctio_list, srr_list_entry) {
- struct qla_tgt_srr_imm *imm, *i, *ti;
- struct qla_tgt_cmd *cmd;
- struct se_cmd *se_cmd;
-
- imm = NULL;
- list_for_each_entry_safe(i, ti, &tgt->srr_imm_list,
- srr_list_entry) {
- if (i->srr_id == sctio->srr_id) {
- list_del(&i->srr_list_entry);
- if (imm) {
- ql_dbg(ql_dbg_tgt_mgt, vha, 0xf068,
- "qla_target(%d): There must be "
- "only one IMM SRR per CTIO SRR "
- "(IMM SRR %p, id %d, CTIO %p\n",
- vha->vp_idx, i, i->srr_id, sctio);
- qlt_reject_free_srr_imm(tgt->vha, i, 0);
- } else
- imm = i;
- }
- }
-
- ql_dbg(ql_dbg_tgt_mgt, vha, 0xf02a,
- "IMM SRR %p, CTIO SRR %p (id %d)\n", imm, sctio,
- sctio->srr_id);
-
- if (imm == NULL) {
- ql_dbg(ql_dbg_tgt_mgt, vha, 0xf02b,
- "Not found matching IMM for SRR CTIO (id %d)\n",
- sctio->srr_id);
- continue;
- } else
- list_del(&sctio->srr_list_entry);
-
- spin_unlock_irqrestore(&tgt->srr_lock, flags);
-
- cmd = sctio->cmd;
- /*
- * Reset qla_tgt_cmd SRR values and SGL pointer+count to follow
- * tcm_qla2xxx_write_pending() and tcm_qla2xxx_queue_data_in()
- * logic..
- */
- cmd->offset = 0;
- if (cmd->free_sg) {
- kfree(cmd->sg);
- cmd->sg = NULL;
- cmd->free_sg = 0;
- }
- se_cmd = &cmd->se_cmd;
-
- cmd->sg_cnt = se_cmd->t_data_nents;
- cmd->sg = se_cmd->t_data_sg;
-
- ql_dbg(ql_dbg_tgt_mgt, vha, 0xf02c,
- "SRR cmd %p (se_cmd %p, tag %lld, op %x), sg_cnt=%d, offset=%d",
- cmd, &cmd->se_cmd, se_cmd->tag, se_cmd->t_task_cdb ?
- se_cmd->t_task_cdb[0] : 0, cmd->sg_cnt, cmd->offset);
-
- qlt_handle_srr(vha, sctio, imm);
-
- kfree(imm);
- kfree(sctio);
- goto restart;
- }
- spin_unlock_irqrestore(&tgt->srr_lock, flags);
-}
-
-/* ha->hardware_lock supposed to be held on entry */
-static void qlt_prepare_srr_imm(struct scsi_qla_host *vha,
- struct imm_ntfy_from_isp *iocb)
-{
- struct qla_tgt_srr_imm *imm;
- struct qla_tgt *tgt = vha->vha_tgt.qla_tgt;
- struct qla_tgt_srr_ctio *sctio;
-
- tgt->imm_srr_id++;
-
- ql_log(ql_log_warn, vha, 0xf02d, "qla_target(%d): SRR received\n",
- vha->vp_idx);
-
- imm = kzalloc(sizeof(*imm), GFP_ATOMIC);
- if (imm != NULL) {
- memcpy(&imm->imm_ntfy, iocb, sizeof(imm->imm_ntfy));
-
- /* IRQ is already OFF */
- spin_lock(&tgt->srr_lock);
- imm->srr_id = tgt->imm_srr_id;
- list_add_tail(&imm->srr_list_entry,
- &tgt->srr_imm_list);
- ql_dbg(ql_dbg_tgt_mgt, vha, 0xf02e,
- "IMM NTFY SRR %p added (id %d, ui %x)\n",
- imm, imm->srr_id, iocb->u.isp24.srr_ui);
- if (tgt->imm_srr_id == tgt->ctio_srr_id) {
- int found = 0;
- list_for_each_entry(sctio, &tgt->srr_ctio_list,
- srr_list_entry) {
- if (sctio->srr_id == imm->srr_id) {
- found = 1;
- break;
- }
- }
- if (found) {
- ql_dbg(ql_dbg_tgt_mgt, vha, 0xf02f, "%s",
- "Scheduling srr work\n");
- schedule_work(&tgt->srr_work);
- } else {
- ql_dbg(ql_dbg_tgt_mgt, vha, 0xf030,
- "qla_target(%d): imm_srr_id "
- "== ctio_srr_id (%d), but there is no "
- "corresponding SRR CTIO, deleting IMM "
- "SRR %p\n", vha->vp_idx, tgt->ctio_srr_id,
- imm);
- list_del(&imm->srr_list_entry);
-
- kfree(imm);
-
- spin_unlock(&tgt->srr_lock);
- goto out_reject;
- }
- }
- spin_unlock(&tgt->srr_lock);
- } else {
- struct qla_tgt_srr_ctio *ts;
-
- ql_dbg(ql_dbg_tgt_mgt, vha, 0xf069,
- "qla_target(%d): Unable to allocate SRR IMM "
- "entry, SRR request will be rejected\n", vha->vp_idx);
-
- /* IRQ is already OFF */
- spin_lock(&tgt->srr_lock);
- list_for_each_entry_safe(sctio, ts, &tgt->srr_ctio_list,
- srr_list_entry) {
- if (sctio->srr_id == tgt->imm_srr_id) {
- ql_dbg(ql_dbg_tgt_mgt, vha, 0xf031,
- "CTIO SRR %p deleted (id %d)\n",
- sctio, sctio->srr_id);
- list_del(&sctio->srr_list_entry);
- qlt_send_term_exchange(vha, sctio->cmd,
- &sctio->cmd->atio, 1, 0);
- kfree(sctio);
- }
- }
- spin_unlock(&tgt->srr_lock);
- goto out_reject;
- }
-
- return;
-
-out_reject:
- qlt_send_notify_ack(vha, iocb, 0, 0, 0,
- NOTIFY_ACK_SRR_FLAGS_REJECT,
- NOTIFY_ACK_SRR_REJECT_REASON_UNABLE_TO_PERFORM,
- NOTIFY_ACK_SRR_FLAGS_REJECT_EXPL_NO_EXPL);
-}
-
/*
* ha->hardware_lock supposed to be held on entry. Might drop it, then reaquire
*/
if (qlt_24xx_handle_els(vha, iocb) == 0)
send_notify_ack = 0;
break;
-
- case IMM_NTFY_SRR:
- qlt_prepare_srr_imm(vha, iocb);
- send_notify_ack = 0;
- break;
-
default:
ql_dbg(ql_dbg_tgt_mgt, vha, 0xf06d,
"qla_target(%d): Received unknown immediate "
struct ctio7_to_24xx *ctio24;
struct qla_hw_data *ha = vha->hw;
request_t *pkt;
- struct qla_tgt_sess *sess = NULL;
+ struct fc_port *sess = NULL;
unsigned long flags;
spin_lock_irqsave(&ha->tgt.sess_lock, flags);
{
struct qla_tgt *tgt = vha->vha_tgt.qla_tgt;
struct qla_hw_data *ha = vha->hw;
- struct qla_tgt_sess *sess;
+ struct fc_port *sess;
struct se_session *se_sess;
struct qla_tgt_cmd *cmd;
int tag;
le16_to_cpu(mailbox[2]), le16_to_cpu(mailbox[3]));
break;
+ case MBA_REJECTED_FCP_CMD:
+ ql_dbg(ql_dbg_tgt_mgt, vha, 0xffff,
+ "qla_target(%d): Async event LS_REJECT occurred "
+ "(m[0]=%x, m[1]=%x, m[2]=%x, m[3]=%x)", vha->vp_idx,
+ le16_to_cpu(mailbox[0]), le16_to_cpu(mailbox[1]),
+ le16_to_cpu(mailbox[2]), le16_to_cpu(mailbox[3]));
+
+ if (le16_to_cpu(mailbox[3]) == 1) {
+ /* exchange starvation. */
+ vha->hw->exch_starvation++;
+ if (vha->hw->exch_starvation > 5) {
+ ql_log(ql_log_warn, vha, 0xffff,
+ "Exchange starvation-. Resetting RISC\n");
+
+ vha->hw->exch_starvation = 0;
+ if (IS_P3P_TYPE(vha->hw))
+ set_bit(FCOE_CTX_RESET_NEEDED,
+ &vha->dpc_flags);
+ else
+ set_bit(ISP_ABORT_NEEDED,
+ &vha->dpc_flags);
+ qla2xxx_wake_dpc(vha);
+ }
+ }
+ break;
+
case MBA_PORT_UPDATE:
ql_dbg(ql_dbg_tgt_mgt, vha, 0xf03d,
"qla_target(%d): Port update async event %#x "
le16_to_cpu(mailbox[2]), le16_to_cpu(mailbox[3]));
login_code = le16_to_cpu(mailbox[2]);
- if (login_code == 0x4)
+ if (login_code == 0x4) {
ql_dbg(ql_dbg_tgt_mgt, vha, 0xf03e,
"Async MB 2: Got PLOGI Complete\n");
- else if (login_code == 0x7)
+ vha->hw->exch_starvation = 0;
+ } else if (login_code == 0x7)
ql_dbg(ql_dbg_tgt_mgt, vha, 0xf03f,
"Async MB 2: Port Logged Out\n");
break;
-
default:
break;
}
static fc_port_t *qlt_get_port_database(struct scsi_qla_host *vha,
uint16_t loop_id)
{
- fc_port_t *fcport;
+ fc_port_t *fcport, *tfcp, *del;
int rc;
+ unsigned long flags;
+ u8 newfcport = 0;
fcport = kzalloc(sizeof(*fcport), GFP_KERNEL);
if (!fcport) {
return NULL;
}
+ del = NULL;
+ spin_lock_irqsave(&vha->hw->tgt.sess_lock, flags);
+ tfcp = qla2x00_find_fcport_by_wwpn(vha, fcport->port_name, 1);
+
+ if (tfcp) {
+ tfcp->d_id = fcport->d_id;
+ tfcp->port_type = fcport->port_type;
+ tfcp->supported_classes = fcport->supported_classes;
+ tfcp->flags |= fcport->flags;
+
+ del = fcport;
+ fcport = tfcp;
+ } else {
+ if (vha->hw->current_topology == ISP_CFG_F)
+ fcport->flags |= FCF_FABRIC_DEVICE;
+
+ list_add_tail(&fcport->list, &vha->vp_fcports);
+ if (!IS_SW_RESV_ADDR(fcport->d_id))
+ vha->fcport_count++;
+ fcport->login_gen++;
+ fcport->disc_state = DSC_LOGIN_COMPLETE;
+ fcport->login_succ = 1;
+ newfcport = 1;
+ }
+
+ fcport->deleted = 0;
+ spin_unlock_irqrestore(&vha->hw->tgt.sess_lock, flags);
+
+ switch (vha->host->active_mode) {
+ case MODE_INITIATOR:
+ case MODE_DUAL:
+ if (newfcport) {
+ if (!IS_IIDMA_CAPABLE(vha->hw) || !vha->hw->flags.gpsc_supported) {
+ ql_dbg(ql_dbg_disc, vha, 0xffff,
+ "%s %d %8phC post upd_fcport fcp_cnt %d\n",
+ __func__, __LINE__, fcport->port_name, vha->fcport_count);
+ qla24xx_post_upd_fcport_work(vha, fcport);
+ } else {
+ ql_dbg(ql_dbg_disc, vha, 0xffff,
+ "%s %d %8phC post gpsc fcp_cnt %d\n",
+ __func__, __LINE__, fcport->port_name, vha->fcport_count);
+ qla24xx_post_gpsc_work(vha, fcport);
+ }
+ }
+ break;
+
+ case MODE_TARGET:
+ default:
+ break;
+ }
+ if (del)
+ qla2x00_free_fcport(del);
+
return fcport;
}
/* Must be called under tgt_mutex */
-static struct qla_tgt_sess *qlt_make_local_sess(struct scsi_qla_host *vha,
+static struct fc_port *qlt_make_local_sess(struct scsi_qla_host *vha,
uint8_t *s_id)
{
- struct qla_tgt_sess *sess = NULL;
+ struct fc_port *sess = NULL;
fc_port_t *fcport = NULL;
int rc, global_resets;
uint16_t loop_id = 0;
+ if ((s_id[0] == 0xFF) && (s_id[1] == 0xFC)) {
+ /*
+ * This is Domain Controller, so it should be
+ * OK to drop SCSI commands from it.
+ */
+ ql_dbg(ql_dbg_tgt_mgt, vha, 0xf042,
+ "Unable to find initiator with S_ID %x:%x:%x",
+ s_id[0], s_id[1], s_id[2]);
+ return NULL;
+ }
+
mutex_lock(&vha->vha_tgt.tgt_mutex);
retry:
if (rc != 0) {
mutex_unlock(&vha->vha_tgt.tgt_mutex);
- if ((s_id[0] == 0xFF) &&
- (s_id[1] == 0xFC)) {
- /*
- * This is Domain Controller, so it should be
- * OK to drop SCSI commands from it.
- */
- ql_dbg(ql_dbg_tgt_mgt, vha, 0xf042,
- "Unable to find initiator with S_ID %x:%x:%x",
- s_id[0], s_id[1], s_id[2]);
- } else
- ql_log(ql_log_info, vha, 0xf071,
- "qla_target(%d): Unable to find "
- "initiator with S_ID %x:%x:%x",
- vha->vp_idx, s_id[0], s_id[1],
- s_id[2]);
+ ql_log(ql_log_info, vha, 0xf071,
+ "qla_target(%d): Unable to find "
+ "initiator with S_ID %x:%x:%x",
+ vha->vp_idx, s_id[0], s_id[1],
+ s_id[2]);
if (rc == -ENOENT) {
qlt_port_logo_t logo;
mutex_unlock(&vha->vha_tgt.tgt_mutex);
- kfree(fcport);
return sess;
}
{
struct scsi_qla_host *vha = tgt->vha;
struct qla_hw_data *ha = vha->hw;
- struct qla_tgt_sess *sess = NULL;
+ struct fc_port *sess = NULL;
unsigned long flags = 0, flags2 = 0;
uint32_t be_s_id;
uint8_t s_id[3];
if (!sess)
goto out_term2;
} else {
- if (sess->deleted == QLA_SESS_DELETION_IN_PROGRESS) {
+ if (sess->deleted) {
sess = NULL;
goto out_term2;
}
- kref_get(&sess->sess_kref);
+ if (!kref_get_unless_zero(&sess->sess_kref)) {
+ ql_dbg(ql_dbg_tgt_tmr, vha, 0xffff,
+ "%s: kref_get fail %8phC \n",
+ __func__, sess->port_name);
+ sess = NULL;
+ goto out_term2;
+ }
}
spin_lock_irqsave(&ha->hardware_lock, flags);
if (rc != 0)
goto out_term;
spin_unlock_irqrestore(&ha->hardware_lock, flags);
-
- qlt_put_sess(sess);
+ if (sess)
+ ha->tgt.tgt_ops->put_sess(sess);
spin_unlock_irqrestore(&ha->tgt.sess_lock, flags2);
return;
qlt_24xx_send_abts_resp(vha, &prm->abts, FCP_TMF_REJECTED, false);
spin_unlock_irqrestore(&ha->hardware_lock, flags);
- qlt_put_sess(sess);
+ if (sess)
+ ha->tgt.tgt_ops->put_sess(sess);
spin_unlock_irqrestore(&ha->tgt.sess_lock, flags2);
}
struct atio_from_isp *a = &prm->tm_iocb2;
struct scsi_qla_host *vha = tgt->vha;
struct qla_hw_data *ha = vha->hw;
- struct qla_tgt_sess *sess = NULL;
+ struct fc_port *sess = NULL;
unsigned long flags;
uint8_t *s_id = NULL; /* to hide compiler warnings */
int rc;
if (!sess)
goto out_term;
} else {
- if (sess->deleted == QLA_SESS_DELETION_IN_PROGRESS) {
+ if (sess->deleted) {
sess = NULL;
goto out_term;
}
- kref_get(&sess->sess_kref);
+ if (!kref_get_unless_zero(&sess->sess_kref)) {
+ ql_dbg(ql_dbg_tgt_tmr, vha, 0xffff,
+ "%s: kref_get fail %8phC\n",
+ __func__, sess->port_name);
+ sess = NULL;
+ goto out_term;
+ }
}
iocb = a;
if (rc != 0)
goto out_term;
- qlt_put_sess(sess);
+ ha->tgt.tgt_ops->put_sess(sess);
spin_unlock_irqrestore(&ha->tgt.sess_lock, flags);
return;
out_term:
qlt_send_term_exchange(vha, NULL, &prm->tm_iocb2, 1, 0);
- qlt_put_sess(sess);
+ ha->tgt.tgt_ops->put_sess(sess);
spin_unlock_irqrestore(&ha->tgt.sess_lock, flags);
}
tgt->ha = ha;
tgt->vha = base_vha;
init_waitqueue_head(&tgt->waitQ);
- INIT_LIST_HEAD(&tgt->sess_list);
INIT_LIST_HEAD(&tgt->del_sess_list);
- INIT_DELAYED_WORK(&tgt->sess_del_work,
- (void (*)(struct work_struct *))qlt_del_sess_work_fn);
spin_lock_init(&tgt->sess_work_lock);
INIT_WORK(&tgt->sess_work, qlt_sess_work_fn);
INIT_LIST_HEAD(&tgt->sess_works_list);
- spin_lock_init(&tgt->srr_lock);
- INIT_LIST_HEAD(&tgt->srr_ctio_list);
- INIT_LIST_HEAD(&tgt->srr_imm_list);
- INIT_WORK(&tgt->srr_work, qlt_handle_srr_work);
atomic_set(&tgt->tgt_global_resets_count, 0);
base_vha->vha_tgt.qla_tgt = tgt;
/* Must be called under HW lock */
static void qlt_set_mode(struct scsi_qla_host *vha)
{
- struct qla_hw_data *ha = vha->hw;
-
switch (ql2x_ini_mode) {
case QLA2XXX_INI_MODE_DISABLED:
case QLA2XXX_INI_MODE_EXCLUSIVE:
vha->host->active_mode = MODE_TARGET;
break;
case QLA2XXX_INI_MODE_ENABLED:
- vha->host->active_mode |= MODE_TARGET;
+ vha->host->active_mode = MODE_UNKNOWN;
+ break;
+ case QLA2XXX_INI_MODE_DUAL:
+ vha->host->active_mode = MODE_DUAL;
break;
default:
break;
}
-
- if (ha->tgt.ini_mode_force_reverse)
- qla_reverse_ini_mode(vha);
}
/* Must be called under HW lock */
static void qlt_clear_mode(struct scsi_qla_host *vha)
{
- struct qla_hw_data *ha = vha->hw;
-
switch (ql2x_ini_mode) {
case QLA2XXX_INI_MODE_DISABLED:
vha->host->active_mode = MODE_UNKNOWN;
vha->host->active_mode = MODE_INITIATOR;
break;
case QLA2XXX_INI_MODE_ENABLED:
- vha->host->active_mode &= ~MODE_TARGET;
+ case QLA2XXX_INI_MODE_DUAL:
+ vha->host->active_mode = MODE_INITIATOR;
break;
default:
break;
}
-
- if (ha->tgt.ini_mode_force_reverse)
- qla_reverse_ini_mode(vha);
}
/*
void
qlt_vport_create(struct scsi_qla_host *vha, struct qla_hw_data *ha)
{
- if (!qla_tgt_mode_enabled(vha))
- return;
-
vha->vha_tgt.qla_tgt = NULL;
mutex_init(&vha->vha_tgt.tgt_mutex);
* FC-4 Feature bit 0 indicates target functionality to the name server.
*/
if (qla_tgt_mode_enabled(vha)) {
- if (qla_ini_mode_enabled(vha))
- ct_req->req.rff_id.fc4_feature = BIT_0 | BIT_1;
- else
- ct_req->req.rff_id.fc4_feature = BIT_0;
+ ct_req->req.rff_id.fc4_feature = BIT_0;
} else if (qla_ini_mode_enabled(vha)) {
ct_req->req.rff_id.fc4_feature = BIT_1;
- }
+ } else if (qla_dual_mode_enabled(vha))
+ ct_req->req.rff_id.fc4_feature = BIT_0 | BIT_1;
}
/*
uint16_t cnt;
struct atio_from_isp *pkt = (struct atio_from_isp *)ha->tgt.atio_ring;
- if (!qla_tgt_mode_enabled(vha))
+ if (qla_ini_mode_enabled(vha))
return;
for (cnt = 0; cnt < ha->tgt.atio_q_length; cnt++) {
qlt_24xx_config_nvram_stage1(struct scsi_qla_host *vha, struct nvram_24xx *nv)
{
struct qla_hw_data *ha = vha->hw;
+ u32 tmp;
+ u16 t;
- if (qla_tgt_mode_enabled(vha)) {
+ if (qla_tgt_mode_enabled(vha) || qla_dual_mode_enabled(vha)) {
if (!ha->tgt.saved_set) {
/* We save only once */
ha->tgt.saved_exchange_count = nv->exchange_count;
ha->tgt.saved_set = 1;
}
- nv->exchange_count = cpu_to_le16(0xFFFF);
+ if (qla_tgt_mode_enabled(vha)) {
+ nv->exchange_count = cpu_to_le16(0xFFFF);
+ } else { /* dual */
+ if (ql_dm_tgt_ex_pct > 100) {
+ ql_dm_tgt_ex_pct = 50;
+ } else if (ql_dm_tgt_ex_pct == 100) {
+ /* leave some for FW */
+ ql_dm_tgt_ex_pct = 95;
+ }
+
+ tmp = ha->orig_fw_xcb_count * ql_dm_tgt_ex_pct;
+ tmp = tmp/100;
+ if (tmp > 0xffff)
+ tmp = 0xffff;
+
+ t = tmp & 0xffff;
+ nv->exchange_count = cpu_to_le16(t);
+ }
/* Enable target mode */
nv->firmware_options_1 |= cpu_to_le32(BIT_4);
/* Disable ini mode, if requested */
- if (!qla_ini_mode_enabled(vha))
+ if (qla_tgt_mode_enabled(vha))
nv->firmware_options_1 |= cpu_to_le32(BIT_5);
/* Disable Full Login after LIP */
qlt_81xx_config_nvram_stage1(struct scsi_qla_host *vha, struct nvram_81xx *nv)
{
struct qla_hw_data *ha = vha->hw;
+ u32 tmp;
+ u16 t;
if (!QLA_TGT_MODE_ENABLED())
return;
- if (qla_tgt_mode_enabled(vha)) {
+ if (qla_tgt_mode_enabled(vha) || qla_dual_mode_enabled(vha)) {
if (!ha->tgt.saved_set) {
/* We save only once */
ha->tgt.saved_exchange_count = nv->exchange_count;
ha->tgt.saved_set = 1;
}
- nv->exchange_count = cpu_to_le16(0xFFFF);
+ if (qla_tgt_mode_enabled(vha)) {
+ nv->exchange_count = cpu_to_le16(0xFFFF);
+ } else { /* dual */
+ if (ql_dm_tgt_ex_pct > 100) {
+ ql_dm_tgt_ex_pct = 50;
+ } else if (ql_dm_tgt_ex_pct == 100) {
+ /* leave some for FW */
+ ql_dm_tgt_ex_pct = 95;
+ }
+
+ tmp = ha->orig_fw_xcb_count * ql_dm_tgt_ex_pct;
+ tmp = tmp/100;
+ if (tmp > 0xffff)
+ tmp = 0xffff;
+ t = tmp & 0xffff;
+ nv->exchange_count = cpu_to_le16(t);
+ }
/* Enable target mode */
nv->firmware_options_1 |= cpu_to_le32(BIT_4);
/* Disable ini mode, if requested */
- if (!qla_ini_mode_enabled(vha))
+ if (qla_tgt_mode_enabled(vha))
nv->firmware_options_1 |= cpu_to_le32(BIT_5);
/* Disable Full Login after LIP */
nv->firmware_options_1 &= cpu_to_le32(~BIT_13);
qlt_modify_vp_config(struct scsi_qla_host *vha,
struct vp_config_entry_24xx *vpmod)
{
- if (qla_tgt_mode_enabled(vha))
+ /* enable target mode. Bit5 = 1 => disable */
+ if (qla_tgt_mode_enabled(vha) || qla_dual_mode_enabled(vha))
vpmod->options_idx1 &= ~BIT_5;
- /* Disable ini mode, if requested */
- if (!qla_ini_mode_enabled(vha))
+
+ /* Disable ini mode, if requested. bit4 = 1 => disable */
+ if (qla_tgt_mode_enabled(vha))
vpmod->options_idx1 &= ~BIT_4;
}
mutex_init(&base_vha->vha_tgt.tgt_mutex);
mutex_init(&base_vha->vha_tgt.tgt_host_action_mutex);
+
+ INIT_LIST_HEAD(&base_vha->unknown_atio_list);
+ INIT_DELAYED_WORK(&base_vha->unknown_atio_work,
+ qlt_unknown_atio_work_fn);
+
qlt_clear_mode(base_vha);
}
ql2x_ini_mode = QLA2XXX_INI_MODE_DISABLED;
else if (strcasecmp(qlini_mode, QLA2XXX_INI_MODE_STR_ENABLED) == 0)
ql2x_ini_mode = QLA2XXX_INI_MODE_ENABLED;
+ else if (strcasecmp(qlini_mode, QLA2XXX_INI_MODE_STR_DUAL) == 0)
+ ql2x_ini_mode = QLA2XXX_INI_MODE_DUAL;
else
return false;
}
qla_tgt_plogi_cachep = kmem_cache_create("qla_tgt_plogi_cachep",
- sizeof(qlt_plogi_ack_t),
- __alignof__(qlt_plogi_ack_t),
- 0, NULL);
+ sizeof(struct qlt_plogi_ack_t), __alignof__(struct qlt_plogi_ack_t),
+ 0, NULL);
if (!qla_tgt_plogi_cachep) {
ql_log(ql_log_fatal, NULL, 0xe06d,
#define QLA2XXX_INI_MODE_STR_EXCLUSIVE "exclusive"
#define QLA2XXX_INI_MODE_STR_DISABLED "disabled"
#define QLA2XXX_INI_MODE_STR_ENABLED "enabled"
+#define QLA2XXX_INI_MODE_STR_DUAL "dual"
#define QLA2XXX_INI_MODE_EXCLUSIVE 0
#define QLA2XXX_INI_MODE_DISABLED 1
#define QLA2XXX_INI_MODE_ENABLED 2
+#define QLA2XXX_INI_MODE_DUAL 3
#define QLA2XXX_COMMAND_COUNT_INIT 250
#define QLA2XXX_IMMED_NOTIFY_COUNT_INIT 250
? le16_to_cpu((iocb)->u.isp2x.target.extended) \
: (uint16_t)(iocb)->u.isp2x.target.id.standard)
-#ifndef IMMED_NOTIFY_TYPE
-#define IMMED_NOTIFY_TYPE 0x0D /* Immediate notify entry. */
-/*
- * ISP queue - immediate notify entry structure definition.
- * This is sent by the ISP to the Target driver.
- * This IOCB would have report of events sent by the
- * initiator, that needs to be handled by the target
- * driver immediately.
- */
-struct imm_ntfy_from_isp {
- uint8_t entry_type; /* Entry type. */
- uint8_t entry_count; /* Entry count. */
- uint8_t sys_define; /* System defined. */
- uint8_t entry_status; /* Entry Status. */
- union {
- struct {
- uint32_t sys_define_2; /* System defined. */
- target_id_t target;
- uint16_t lun;
- uint8_t target_id;
- uint8_t reserved_1;
- uint16_t status_modifier;
- uint16_t status;
- uint16_t task_flags;
- uint16_t seq_id;
- uint16_t srr_rx_id;
- uint32_t srr_rel_offs;
- uint16_t srr_ui;
-#define SRR_IU_DATA_IN 0x1
-#define SRR_IU_DATA_OUT 0x5
-#define SRR_IU_STATUS 0x7
- uint16_t srr_ox_id;
- uint8_t reserved_2[28];
- } isp2x;
- struct {
- uint32_t reserved;
- uint16_t nport_handle;
- uint16_t reserved_2;
- uint16_t flags;
-#define NOTIFY24XX_FLAGS_GLOBAL_TPRLO BIT_1
-#define NOTIFY24XX_FLAGS_PUREX_IOCB BIT_0
- uint16_t srr_rx_id;
- uint16_t status;
- uint8_t status_subcode;
- uint8_t fw_handle;
- uint32_t exchange_address;
- uint32_t srr_rel_offs;
- uint16_t srr_ui;
- uint16_t srr_ox_id;
- union {
- struct {
- uint8_t node_name[8];
- } plogi; /* PLOGI/ADISC/PDISC */
- struct {
- /* PRLI word 3 bit 0-15 */
- uint16_t wd3_lo;
- uint8_t resv0[6];
- } prli;
- struct {
- uint8_t port_id[3];
- uint8_t resv1;
- uint16_t nport_handle;
- uint16_t resv2;
- } req_els;
- } u;
- uint8_t port_name[8];
- uint8_t resv3[3];
- uint8_t vp_index;
- uint32_t reserved_5;
- uint8_t port_id[3];
- uint8_t reserved_6;
- } isp24;
- } u;
- uint16_t reserved_7;
- uint16_t ox_id;
-} __packed;
-#endif
-
#ifndef NOTIFY_ACK_TYPE
#define NOTIFY_ACK_TYPE 0x0E /* Notify acknowledge entry. */
/*
\********************************************************************/
struct qla_tgt_mgmt_cmd;
-struct qla_tgt_sess;
+struct fc_port;
/*
* This structure provides a template of function calls that the
unsigned char *, uint32_t, int, int, int);
void (*handle_data)(struct qla_tgt_cmd *);
void (*handle_dif_err)(struct qla_tgt_cmd *);
- int (*handle_tmr)(struct qla_tgt_mgmt_cmd *, uint32_t, uint8_t,
+ int (*handle_tmr)(struct qla_tgt_mgmt_cmd *, uint32_t, uint16_t,
uint32_t);
void (*free_cmd)(struct qla_tgt_cmd *);
void (*free_mcmd)(struct qla_tgt_mgmt_cmd *);
- void (*free_session)(struct qla_tgt_sess *);
+ void (*free_session)(struct fc_port *);
int (*check_initiator_node_acl)(struct scsi_qla_host *, unsigned char *,
- struct qla_tgt_sess *);
- void (*update_sess)(struct qla_tgt_sess *, port_id_t, uint16_t, bool);
- struct qla_tgt_sess *(*find_sess_by_loop_id)(struct scsi_qla_host *,
+ struct fc_port *);
+ void (*update_sess)(struct fc_port *, port_id_t, uint16_t, bool);
+ struct fc_port *(*find_sess_by_loop_id)(struct scsi_qla_host *,
const uint16_t);
- struct qla_tgt_sess *(*find_sess_by_s_id)(struct scsi_qla_host *,
+ struct fc_port *(*find_sess_by_s_id)(struct scsi_qla_host *,
const uint8_t *);
- void (*clear_nacl_from_fcport_map)(struct qla_tgt_sess *);
- void (*shutdown_sess)(struct qla_tgt_sess *);
+ void (*clear_nacl_from_fcport_map)(struct fc_port *);
+ void (*put_sess)(struct fc_port *);
+ void (*shutdown_sess)(struct fc_port *);
};
int qla2x00_wait_for_hba_online(struct scsi_qla_host *);
#define QLA_TGT_ABORT_ALL 0xFFFE
#define QLA_TGT_NEXUS_LOSS_SESS 0xFFFD
#define QLA_TGT_NEXUS_LOSS 0xFFFC
+#define QLA_TGT_ABTS 0xFFFB
+#define QLA_TGT_2G_ABORT_TASK 0xFFFA
/* Notify Acknowledge flags */
#define NOTIFY_ACK_RES_COUNT BIT_8
/* Count of sessions refering qla_tgt. Protected by hardware_lock. */
int sess_count;
- /* Protected by hardware_lock. Addition also protected by tgt_mutex. */
- struct list_head sess_list;
-
/* Protected by hardware_lock */
struct list_head del_sess_list;
- struct delayed_work sess_del_work;
spinlock_t sess_work_lock;
struct list_head sess_works_list;
int notify_ack_expected;
int abts_resp_expected;
int modify_lun_expected;
-
- int ctio_srr_id;
- int imm_srr_id;
- spinlock_t srr_lock;
- struct list_head srr_ctio_list;
- struct list_head srr_imm_list;
- struct work_struct srr_work;
-
atomic_t tgt_global_resets_count;
-
struct list_head tgt_list_entry;
};
bool aborted;
};
-enum qla_sess_deletion {
- QLA_SESS_DELETION_NONE = 0,
- QLA_SESS_DELETION_PENDING = 1, /* hopefully we can get rid of
- * this one */
- QLA_SESS_DELETION_IN_PROGRESS = 2,
-};
-
-typedef enum {
- QLT_PLOGI_LINK_SAME_WWN,
- QLT_PLOGI_LINK_CONFLICT,
- QLT_PLOGI_LINK_MAX
-} qlt_plogi_link_t;
-
-typedef struct {
- struct list_head list;
- struct imm_ntfy_from_isp iocb;
- port_id_t id;
- int ref_count;
-} qlt_plogi_ack_t;
-
-/*
- * Equivilant to IT Nexus (Initiator-Target)
- */
-struct qla_tgt_sess {
- uint16_t loop_id;
- port_id_t s_id;
-
- unsigned int conf_compl_supported:1;
- unsigned int deleted:2;
- unsigned int local:1;
- unsigned int logout_on_delete:1;
- unsigned int keep_nport_handle:1;
- unsigned int send_els_logo:1;
-
- unsigned char logout_completed;
-
- int generation;
-
- struct se_session *se_sess;
- struct kref sess_kref;
- struct scsi_qla_host *vha;
- struct qla_tgt *tgt;
-
- struct list_head sess_list_entry;
- unsigned long expires;
- struct list_head del_list_entry;
-
- uint8_t port_name[WWN_SIZE];
- struct work_struct free_work;
-
- qlt_plogi_ack_t *plogi_link[QLT_PLOGI_LINK_MAX];
+enum trace_flags {
+ TRC_NEW_CMD = BIT_0,
+ TRC_DO_WORK = BIT_1,
+ TRC_DO_WORK_ERR = BIT_2,
+ TRC_XFR_RDY = BIT_3,
+ TRC_XMIT_DATA = BIT_4,
+ TRC_XMIT_STATUS = BIT_5,
+ TRC_SRR_RSP = BIT_6,
+ TRC_SRR_XRDY = BIT_7,
+ TRC_SRR_TERM = BIT_8,
+ TRC_SRR_CTIO = BIT_9,
+ TRC_FLUSH = BIT_10,
+ TRC_CTIO_ERR = BIT_11,
+ TRC_CTIO_DONE = BIT_12,
+ TRC_CTIO_ABORTED = BIT_13,
+ TRC_CTIO_STRANGE= BIT_14,
+ TRC_CMD_DONE = BIT_15,
+ TRC_CMD_CHK_STOP = BIT_16,
+ TRC_CMD_FREE = BIT_17,
+ TRC_DATA_IN = BIT_18,
+ TRC_ABORT = BIT_19,
};
-typedef enum {
- /*
- * BIT_0 - Atio Arrival / schedule to work
- * BIT_1 - qlt_do_work
- * BIT_2 - qlt_do work failed
- * BIT_3 - xfer rdy/tcm_qla2xxx_write_pending
- * BIT_4 - read respond/tcm_qla2xx_queue_data_in
- * BIT_5 - status respond / tcm_qla2xx_queue_status
- * BIT_6 - tcm request to abort/Term exchange.
- * pre_xmit_response->qlt_send_term_exchange
- * BIT_7 - SRR received (qlt_handle_srr->qlt_xmit_response)
- * BIT_8 - SRR received (qlt_handle_srr->qlt_rdy_to_xfer)
- * BIT_9 - SRR received (qla_handle_srr->qlt_send_term_exchange)
- * BIT_10 - Data in - hanlde_data->tcm_qla2xxx_handle_data
-
- * BIT_12 - good completion - qlt_ctio_do_completion -->free_cmd
- * BIT_13 - Bad completion -
- * qlt_ctio_do_completion --> qlt_term_ctio_exchange
- * BIT_14 - Back end data received/sent.
- * BIT_15 - SRR prepare ctio
- * BIT_16 - complete free
- * BIT_17 - flush - qlt_abort_cmd_on_host_reset
- * BIT_18 - completion w/abort status
- * BIT_19 - completion w/unknown status
- * BIT_20 - tcm_qla2xxx_free_cmd
- */
- CMD_FLAG_DATA_WORK = BIT_11,
- CMD_FLAG_DATA_WORK_FREE = BIT_21,
-} cmd_flags_t;
-
struct qla_tgt_cmd {
struct se_cmd se_cmd;
- struct qla_tgt_sess *sess;
+ struct fc_port *sess;
int state;
struct work_struct free_work;
struct work_struct work;
unsigned int cmd_sent_to_fw:1;
unsigned int cmd_in_wq:1;
unsigned int aborted:1;
+ unsigned int data_work:1;
+ unsigned int data_work_free:1;
struct scatterlist *sg; /* cmd data buffer SG vector */
int sg_cnt; /* SG segments count */
uint64_t jiffies_at_alloc;
uint64_t jiffies_at_free;
- cmd_flags_t cmd_flags;
+ enum trace_flags trc_flags;
};
struct qla_tgt_sess_work_param {
};
struct qla_tgt_mgmt_cmd {
- uint8_t tmr_func;
+ uint16_t tmr_func;
uint8_t fc_tm_rsp;
- struct qla_tgt_sess *sess;
+ struct fc_port *sess;
struct se_cmd se_cmd;
struct work_struct free_work;
unsigned int flags;
uint16_t tot_dsds;
};
-struct qla_tgt_srr_imm {
- struct list_head srr_list_entry;
- int srr_id;
- struct imm_ntfy_from_isp imm_ntfy;
-};
-
-struct qla_tgt_srr_ctio {
- struct list_head srr_list_entry;
- int srr_id;
- struct qla_tgt_cmd *cmd;
-};
-
/* Check for Switch reserved address */
#define IS_SW_RESV_ADDR(_s_id) \
((_s_id.b.domain == 0xff) && (_s_id.b.area == 0xfc))
extern int qlt_lport_register(void *, u64, u64, u64,
int (*callback)(struct scsi_qla_host *, void *, u64, u64));
extern void qlt_lport_deregister(struct scsi_qla_host *);
-void qlt_put_sess(struct qla_tgt_sess *sess);
+extern void qlt_unreg_sess(struct fc_port *);
extern void qlt_fc_port_added(struct scsi_qla_host *, fc_port_t *);
extern void qlt_fc_port_deleted(struct scsi_qla_host *, fc_port_t *, int);
extern int __init qlt_init(void);
* is not set. Right now, ha value is ignored.
*/
#define QLA_TGT_MODE_ENABLED() (ql2x_ini_mode != QLA2XXX_INI_MODE_ENABLED)
+
extern int ql2x_ini_mode;
static inline bool qla_tgt_mode_enabled(struct scsi_qla_host *ha)
{
- return ha->host->active_mode & MODE_TARGET;
+ return ha->host->active_mode == MODE_TARGET;
}
static inline bool qla_ini_mode_enabled(struct scsi_qla_host *ha)
{
- return ha->host->active_mode & MODE_INITIATOR;
+ return ha->host->active_mode == MODE_INITIATOR;
}
-static inline void qla_reverse_ini_mode(struct scsi_qla_host *ha)
+static inline bool qla_dual_mode_enabled(struct scsi_qla_host *ha)
{
- if (ha->host->active_mode & MODE_INITIATOR)
- ha->host->active_mode &= ~MODE_INITIATOR;
- else
- ha->host->active_mode |= MODE_INITIATOR;
+ return (ha->host->active_mode == MODE_DUAL);
}
static inline uint32_t sid_to_key(const uint8_t *s_id)
cmd->cmd_in_wq = 0;
- WARN_ON(cmd->cmd_flags & BIT_16);
+ WARN_ON(cmd->trc_flags & TRC_CMD_FREE);
cmd->vha->tgt_counters.qla_core_ret_sta_ctio++;
- cmd->cmd_flags |= BIT_16;
+ cmd->trc_flags |= TRC_CMD_FREE;
transport_generic_free_cmd(&cmd->se_cmd, 0);
}
cmd->vha->tgt_counters.core_qla_free_cmd++;
cmd->cmd_in_wq = 1;
- BUG_ON(cmd->cmd_flags & BIT_20);
- cmd->cmd_flags |= BIT_20;
+ WARN_ON(cmd->trc_flags & TRC_CMD_DONE);
+ cmd->trc_flags |= TRC_CMD_DONE;
INIT_WORK(&cmd->work, tcm_qla2xxx_complete_free);
queue_work_on(smp_processor_id(), tcm_qla2xxx_free_wq, &cmd->work);
if ((se_cmd->se_cmd_flags & SCF_SCSI_TMR_CDB) == 0) {
cmd = container_of(se_cmd, struct qla_tgt_cmd, se_cmd);
- cmd->cmd_flags |= BIT_14;
+ cmd->trc_flags |= TRC_CMD_CHK_STOP;
}
return target_put_sess_cmd(se_cmd);
qlt_free_cmd(cmd);
}
+static void tcm_qla2xxx_release_session(struct kref *kref)
+{
+ struct fc_port *sess = container_of(kref,
+ struct fc_port, sess_kref);
+
+ qlt_unreg_sess(sess);
+}
+
+static void tcm_qla2xxx_put_sess(struct fc_port *sess)
+{
+ if (!sess)
+ return;
+
+ assert_spin_locked(&sess->vha->hw->tgt.sess_lock);
+ kref_put(&sess->sess_kref, tcm_qla2xxx_release_session);
+}
+
static void tcm_qla2xxx_close_session(struct se_session *se_sess)
{
- struct qla_tgt_sess *sess = se_sess->fabric_sess_ptr;
+ struct fc_port *sess = se_sess->fabric_sess_ptr;
struct scsi_qla_host *vha;
unsigned long flags;
spin_lock_irqsave(&vha->hw->tgt.sess_lock, flags);
target_sess_cmd_list_set_waiting(se_sess);
- qlt_put_sess(sess);
+ tcm_qla2xxx_put_sess(sess);
spin_unlock_irqrestore(&vha->hw->tgt.sess_lock, flags);
}
cmd->se_cmd.se_cmd_flags);
return 0;
}
- cmd->cmd_flags |= BIT_3;
+ cmd->trc_flags |= TRC_XFR_RDY;
cmd->bufflen = se_cmd->data_length;
cmd->dma_data_direction = target_reverse_dma_direction(se_cmd);
{
struct se_cmd *se_cmd = &cmd->se_cmd;
struct se_session *se_sess;
- struct qla_tgt_sess *sess;
+ struct fc_port *sess;
#ifdef CONFIG_TCM_QLA2XXX_DEBUG
struct se_portal_group *se_tpg;
struct tcm_qla2xxx_tpg *tpg;
sess = cmd->sess;
if (!sess) {
- pr_err("Unable to locate struct qla_tgt_sess from qla_tgt_cmd\n");
+ pr_err("Unable to locate struct fc_port from qla_tgt_cmd\n");
return -EINVAL;
}
cmd->cmd_in_wq = 0;
spin_lock_irqsave(&cmd->cmd_lock, flags);
- cmd->cmd_flags |= CMD_FLAG_DATA_WORK;
+ cmd->data_work = 1;
if (cmd->aborted) {
- cmd->cmd_flags |= CMD_FLAG_DATA_WORK_FREE;
+ cmd->data_work_free = 1;
spin_unlock_irqrestore(&cmd->cmd_lock, flags);
tcm_qla2xxx_free_cmd(cmd);
*/
static void tcm_qla2xxx_handle_data(struct qla_tgt_cmd *cmd)
{
- cmd->cmd_flags |= BIT_10;
+ cmd->trc_flags |= TRC_DATA_IN;
cmd->cmd_in_wq = 1;
INIT_WORK(&cmd->work, tcm_qla2xxx_handle_data_work);
queue_work_on(smp_processor_id(), tcm_qla2xxx_free_wq, &cmd->work);
* Called from qla_target.c:qlt_issue_task_mgmt()
*/
static int tcm_qla2xxx_handle_tmr(struct qla_tgt_mgmt_cmd *mcmd, uint32_t lun,
- uint8_t tmr_func, uint32_t tag)
+ uint16_t tmr_func, uint32_t tag)
{
- struct qla_tgt_sess *sess = mcmd->sess;
+ struct fc_port *sess = mcmd->sess;
struct se_cmd *se_cmd = &mcmd->se_cmd;
+ int transl_tmr_func = 0;
+
+ switch (tmr_func) {
+ case QLA_TGT_ABTS:
+ pr_debug("%ld: ABTS received\n", sess->vha->host_no);
+ transl_tmr_func = TMR_ABORT_TASK;
+ break;
+ case QLA_TGT_2G_ABORT_TASK:
+ pr_debug("%ld: 2G Abort Task received\n", sess->vha->host_no);
+ transl_tmr_func = TMR_ABORT_TASK;
+ break;
+ case QLA_TGT_CLEAR_ACA:
+ pr_debug("%ld: CLEAR_ACA received\n", sess->vha->host_no);
+ transl_tmr_func = TMR_CLEAR_ACA;
+ break;
+ case QLA_TGT_TARGET_RESET:
+ pr_debug("%ld: TARGET_RESET received\n", sess->vha->host_no);
+ transl_tmr_func = TMR_TARGET_WARM_RESET;
+ break;
+ case QLA_TGT_LUN_RESET:
+ pr_debug("%ld: LUN_RESET received\n", sess->vha->host_no);
+ transl_tmr_func = TMR_LUN_RESET;
+ break;
+ case QLA_TGT_CLEAR_TS:
+ pr_debug("%ld: CLEAR_TS received\n", sess->vha->host_no);
+ transl_tmr_func = TMR_CLEAR_TASK_SET;
+ break;
+ case QLA_TGT_ABORT_TS:
+ pr_debug("%ld: ABORT_TS received\n", sess->vha->host_no);
+ transl_tmr_func = TMR_ABORT_TASK_SET;
+ break;
+ default:
+ pr_debug("%ld: Unknown task mgmt fn 0x%x\n",
+ sess->vha->host_no, tmr_func);
+ return -ENOSYS;
+ }
return target_submit_tmr(se_cmd, sess->se_sess, NULL, lun, mcmd,
- tmr_func, GFP_ATOMIC, tag, TARGET_SCF_ACK_KREF);
+ transl_tmr_func, GFP_ATOMIC, tag, TARGET_SCF_ACK_KREF);
}
static int tcm_qla2xxx_queue_data_in(struct se_cmd *se_cmd)
return 0;
}
- cmd->cmd_flags |= BIT_4;
+ cmd->trc_flags |= TRC_XMIT_DATA;
cmd->bufflen = se_cmd->data_length;
cmd->dma_data_direction = target_reverse_dma_direction(se_cmd);
cmd->sg_cnt = 0;
cmd->offset = 0;
cmd->dma_data_direction = target_reverse_dma_direction(se_cmd);
- if (cmd->cmd_flags & BIT_5) {
- pr_crit("Bit_5 already set for cmd = %p.\n", cmd);
+ if (cmd->trc_flags & TRC_XMIT_STATUS) {
+ pr_crit("Multiple calls for status = %p.\n", cmd);
dump_stack();
}
- cmd->cmd_flags |= BIT_5;
+ cmd->trc_flags |= TRC_XMIT_STATUS;
if (se_cmd->data_direction == DMA_FROM_DEVICE) {
/*
qlt_xmit_tm_rsp(mcmd);
}
-
-#define DATA_WORK_NOT_FREE(_flags) \
- (( _flags & (CMD_FLAG_DATA_WORK|CMD_FLAG_DATA_WORK_FREE)) == \
- CMD_FLAG_DATA_WORK)
+#define DATA_WORK_NOT_FREE(_cmd) (_cmd->data_work && !_cmd->data_work_free)
static void tcm_qla2xxx_aborted_task(struct se_cmd *se_cmd)
{
struct qla_tgt_cmd *cmd = container_of(se_cmd,
spin_lock_irqsave(&cmd->cmd_lock, flags);
if ((cmd->state == QLA_TGT_STATE_NEW)||
- ((cmd->state == QLA_TGT_STATE_DATA_IN) &&
- DATA_WORK_NOT_FREE(cmd->cmd_flags)) ) {
-
- cmd->cmd_flags |= CMD_FLAG_DATA_WORK_FREE;
+ ((cmd->state == QLA_TGT_STATE_DATA_IN) &&
+ DATA_WORK_NOT_FREE(cmd))) {
+ cmd->data_work_free = 1;
spin_unlock_irqrestore(&cmd->cmd_lock, flags);
- /* Cmd have not reached firmware.
- * Use this trigger to free it. */
+ /*
+ * cmd has not reached fw, Use this trigger to free it.
+ */
tcm_qla2xxx_free_cmd(cmd);
return;
}
}
static void tcm_qla2xxx_clear_sess_lookup(struct tcm_qla2xxx_lport *,
- struct tcm_qla2xxx_nacl *, struct qla_tgt_sess *);
+ struct tcm_qla2xxx_nacl *, struct fc_port *);
/*
* Expected to be called with struct qla_hw_data->tgt.sess_lock held
*/
-static void tcm_qla2xxx_clear_nacl_from_fcport_map(struct qla_tgt_sess *sess)
+static void tcm_qla2xxx_clear_nacl_from_fcport_map(struct fc_port *sess)
{
struct se_node_acl *se_nacl = sess->se_sess->se_node_acl;
struct se_portal_group *se_tpg = se_nacl->se_tpg;
tcm_qla2xxx_clear_sess_lookup(lport, nacl, sess);
}
-static void tcm_qla2xxx_shutdown_sess(struct qla_tgt_sess *sess)
+static void tcm_qla2xxx_shutdown_sess(struct fc_port *sess)
{
assert_spin_locked(&sess->vha->hw->tgt.sess_lock);
target_sess_cmd_list_set_waiting(sess->se_sess);
/*
* Expected to be called with struct qla_hw_data->tgt.sess_lock held
*/
-static struct qla_tgt_sess *tcm_qla2xxx_find_sess_by_s_id(
+static struct fc_port *tcm_qla2xxx_find_sess_by_s_id(
scsi_qla_host_t *vha,
const uint8_t *s_id)
{
se_nacl, se_nacl->initiatorname);
nacl = container_of(se_nacl, struct tcm_qla2xxx_nacl, se_node_acl);
- if (!nacl->qla_tgt_sess) {
- pr_err("Unable to locate struct qla_tgt_sess\n");
+ if (!nacl->fc_port) {
+ pr_err("Unable to locate struct fc_port\n");
return NULL;
}
- return nacl->qla_tgt_sess;
+ return nacl->fc_port;
}
/*
struct se_node_acl *new_se_nacl,
struct tcm_qla2xxx_nacl *nacl,
struct se_session *se_sess,
- struct qla_tgt_sess *qla_tgt_sess,
+ struct fc_port *fc_port,
uint8_t *s_id)
{
u32 key;
pr_debug("Wiping nonexisting fc_port entry\n");
}
- qla_tgt_sess->se_sess = se_sess;
- nacl->qla_tgt_sess = qla_tgt_sess;
+ fc_port->se_sess = se_sess;
+ nacl->fc_port = fc_port;
return;
}
- if (nacl->qla_tgt_sess) {
+ if (nacl->fc_port) {
if (new_se_nacl == NULL) {
- pr_debug("Clearing existing nacl->qla_tgt_sess and fc_port entry\n");
+ pr_debug("Clearing existing nacl->fc_port and fc_port entry\n");
btree_remove32(&lport->lport_fcport_map, key);
- nacl->qla_tgt_sess = NULL;
+ nacl->fc_port = NULL;
return;
}
- pr_debug("Replacing existing nacl->qla_tgt_sess and fc_port entry\n");
+ pr_debug("Replacing existing nacl->fc_port and fc_port entry\n");
btree_update32(&lport->lport_fcport_map, key, new_se_nacl);
- qla_tgt_sess->se_sess = se_sess;
- nacl->qla_tgt_sess = qla_tgt_sess;
+ fc_port->se_sess = se_sess;
+ nacl->fc_port = fc_port;
return;
}
return;
}
- pr_debug("Replacing existing fc_port entry w/o active nacl->qla_tgt_sess\n");
+ pr_debug("Replacing existing fc_port entry w/o active nacl->fc_port\n");
btree_update32(&lport->lport_fcport_map, key, new_se_nacl);
- qla_tgt_sess->se_sess = se_sess;
- nacl->qla_tgt_sess = qla_tgt_sess;
+ fc_port->se_sess = se_sess;
+ nacl->fc_port = fc_port;
- pr_debug("Setup nacl->qla_tgt_sess %p by s_id for se_nacl: %p, initiatorname: %s\n",
- nacl->qla_tgt_sess, new_se_nacl, new_se_nacl->initiatorname);
+ pr_debug("Setup nacl->fc_port %p by s_id for se_nacl: %p, initiatorname: %s\n",
+ nacl->fc_port, new_se_nacl, new_se_nacl->initiatorname);
}
/*
* Expected to be called with struct qla_hw_data->tgt.sess_lock held
*/
-static struct qla_tgt_sess *tcm_qla2xxx_find_sess_by_loop_id(
+static struct fc_port *tcm_qla2xxx_find_sess_by_loop_id(
scsi_qla_host_t *vha,
const uint16_t loop_id)
{
nacl = container_of(se_nacl, struct tcm_qla2xxx_nacl, se_node_acl);
- if (!nacl->qla_tgt_sess) {
- pr_err("Unable to locate struct qla_tgt_sess\n");
+ if (!nacl->fc_port) {
+ pr_err("Unable to locate struct fc_port\n");
return NULL;
}
- return nacl->qla_tgt_sess;
+ return nacl->fc_port;
}
/*
struct se_node_acl *new_se_nacl,
struct tcm_qla2xxx_nacl *nacl,
struct se_session *se_sess,
- struct qla_tgt_sess *qla_tgt_sess,
+ struct fc_port *fc_port,
uint16_t loop_id)
{
struct se_node_acl *saved_nacl;
if (!saved_nacl) {
pr_debug("Setting up new fc_loopid->se_nacl to new_se_nacl\n");
fc_loopid->se_nacl = new_se_nacl;
- if (qla_tgt_sess->se_sess != se_sess)
- qla_tgt_sess->se_sess = se_sess;
- if (nacl->qla_tgt_sess != qla_tgt_sess)
- nacl->qla_tgt_sess = qla_tgt_sess;
+ if (fc_port->se_sess != se_sess)
+ fc_port->se_sess = se_sess;
+ if (nacl->fc_port != fc_port)
+ nacl->fc_port = fc_port;
return;
}
- if (nacl->qla_tgt_sess) {
+ if (nacl->fc_port) {
if (new_se_nacl == NULL) {
- pr_debug("Clearing nacl->qla_tgt_sess and fc_loopid->se_nacl\n");
+ pr_debug("Clearing nacl->fc_port and fc_loopid->se_nacl\n");
fc_loopid->se_nacl = NULL;
- nacl->qla_tgt_sess = NULL;
+ nacl->fc_port = NULL;
return;
}
- pr_debug("Replacing existing nacl->qla_tgt_sess and fc_loopid->se_nacl\n");
+ pr_debug("Replacing existing nacl->fc_port and fc_loopid->se_nacl\n");
fc_loopid->se_nacl = new_se_nacl;
- if (qla_tgt_sess->se_sess != se_sess)
- qla_tgt_sess->se_sess = se_sess;
- if (nacl->qla_tgt_sess != qla_tgt_sess)
- nacl->qla_tgt_sess = qla_tgt_sess;
+ if (fc_port->se_sess != se_sess)
+ fc_port->se_sess = se_sess;
+ if (nacl->fc_port != fc_port)
+ nacl->fc_port = fc_port;
return;
}
return;
}
- pr_debug("Replacing existing fc_loopid->se_nacl w/o active nacl->qla_tgt_sess\n");
+ pr_debug("Replacing existing fc_loopid->se_nacl w/o active nacl->fc_port\n");
fc_loopid->se_nacl = new_se_nacl;
- if (qla_tgt_sess->se_sess != se_sess)
- qla_tgt_sess->se_sess = se_sess;
- if (nacl->qla_tgt_sess != qla_tgt_sess)
- nacl->qla_tgt_sess = qla_tgt_sess;
+ if (fc_port->se_sess != se_sess)
+ fc_port->se_sess = se_sess;
+ if (nacl->fc_port != fc_port)
+ nacl->fc_port = fc_port;
- pr_debug("Setup nacl->qla_tgt_sess %p by loop_id for se_nacl: %p, initiatorname: %s\n",
- nacl->qla_tgt_sess, new_se_nacl, new_se_nacl->initiatorname);
+ pr_debug("Setup nacl->fc_port %p by loop_id for se_nacl: %p, initiatorname: %s\n",
+ nacl->fc_port, new_se_nacl, new_se_nacl->initiatorname);
}
/*
* Should always be called with qla_hw_data->tgt.sess_lock held.
*/
static void tcm_qla2xxx_clear_sess_lookup(struct tcm_qla2xxx_lport *lport,
- struct tcm_qla2xxx_nacl *nacl, struct qla_tgt_sess *sess)
+ struct tcm_qla2xxx_nacl *nacl, struct fc_port *sess)
{
struct se_session *se_sess = sess->se_sess;
unsigned char be_sid[3];
- be_sid[0] = sess->s_id.b.domain;
- be_sid[1] = sess->s_id.b.area;
- be_sid[2] = sess->s_id.b.al_pa;
+ be_sid[0] = sess->d_id.b.domain;
+ be_sid[1] = sess->d_id.b.area;
+ be_sid[2] = sess->d_id.b.al_pa;
tcm_qla2xxx_set_sess_by_s_id(lport, NULL, nacl, se_sess,
sess, be_sid);
sess, sess->loop_id);
}
-static void tcm_qla2xxx_free_session(struct qla_tgt_sess *sess)
+static void tcm_qla2xxx_free_session(struct fc_port *sess)
{
struct qla_tgt *tgt = sess->tgt;
struct qla_hw_data *ha = tgt->ha;
se_sess = sess->se_sess;
if (!se_sess) {
- pr_err("struct qla_tgt_sess->se_sess is NULL\n");
+ pr_err("struct fc_port->se_sess is NULL\n");
dump_stack();
return;
}
struct se_node_acl *se_nacl = se_sess->se_node_acl;
struct tcm_qla2xxx_nacl *nacl = container_of(se_nacl,
struct tcm_qla2xxx_nacl, se_node_acl);
- struct qla_tgt_sess *qlat_sess = p;
+ struct fc_port *qlat_sess = p;
uint16_t loop_id = qlat_sess->loop_id;
unsigned long flags;
unsigned char be_sid[3];
- be_sid[0] = qlat_sess->s_id.b.domain;
- be_sid[1] = qlat_sess->s_id.b.area;
- be_sid[2] = qlat_sess->s_id.b.al_pa;
+ be_sid[0] = qlat_sess->d_id.b.domain;
+ be_sid[1] = qlat_sess->d_id.b.area;
+ be_sid[2] = qlat_sess->d_id.b.al_pa;
/*
* And now setup se_nacl and session pointers into HW lport internal
static int tcm_qla2xxx_check_initiator_node_acl(
scsi_qla_host_t *vha,
unsigned char *fc_wwpn,
- struct qla_tgt_sess *qlat_sess)
+ struct fc_port *qlat_sess)
{
struct qla_hw_data *ha = vha->hw;
struct tcm_qla2xxx_lport *lport;
return 0;
}
-static void tcm_qla2xxx_update_sess(struct qla_tgt_sess *sess, port_id_t s_id,
+static void tcm_qla2xxx_update_sess(struct fc_port *sess, port_id_t s_id,
uint16_t loop_id, bool conf_compl_supported)
{
struct qla_tgt *tgt = sess->tgt;
u32 key;
- if (sess->loop_id != loop_id || sess->s_id.b24 != s_id.b24)
+ if (sess->loop_id != loop_id || sess->d_id.b24 != s_id.b24)
pr_info("Updating session %p from port %8phC loop_id %d -> %d s_id %x:%x:%x -> %x:%x:%x\n",
sess, sess->port_name,
- sess->loop_id, loop_id, sess->s_id.b.domain,
- sess->s_id.b.area, sess->s_id.b.al_pa, s_id.b.domain,
+ sess->loop_id, loop_id, sess->d_id.b.domain,
+ sess->d_id.b.area, sess->d_id.b.al_pa, s_id.b.domain,
s_id.b.area, s_id.b.al_pa);
if (sess->loop_id != loop_id) {
sess->loop_id = loop_id;
}
- if (sess->s_id.b24 != s_id.b24) {
- key = (((u32) sess->s_id.b.domain << 16) |
- ((u32) sess->s_id.b.area << 8) |
- ((u32) sess->s_id.b.al_pa));
+ if (sess->d_id.b24 != s_id.b24) {
+ key = (((u32) sess->d_id.b.domain << 16) |
+ ((u32) sess->d_id.b.area << 8) |
+ ((u32) sess->d_id.b.al_pa));
if (btree_lookup32(&lport->lport_fcport_map, key))
- WARN(btree_remove32(&lport->lport_fcport_map, key) != se_nacl,
- "Found wrong se_nacl when updating s_id %x:%x:%x\n",
- sess->s_id.b.domain, sess->s_id.b.area, sess->s_id.b.al_pa);
+ WARN(btree_remove32(&lport->lport_fcport_map, key) !=
+ se_nacl, "Found wrong se_nacl when updating s_id %x:%x:%x\n",
+ sess->d_id.b.domain, sess->d_id.b.area,
+ sess->d_id.b.al_pa);
else
WARN(1, "No lport_fcport_map entry for s_id %x:%x:%x\n",
- sess->s_id.b.domain, sess->s_id.b.area, sess->s_id.b.al_pa);
+ sess->d_id.b.domain, sess->d_id.b.area,
+ sess->d_id.b.al_pa);
key = (((u32) s_id.b.domain << 16) |
((u32) s_id.b.area << 8) |
s_id.b.domain, s_id.b.area, s_id.b.al_pa);
btree_update32(&lport->lport_fcport_map, key, se_nacl);
} else {
- btree_insert32(&lport->lport_fcport_map, key, se_nacl, GFP_ATOMIC);
+ btree_insert32(&lport->lport_fcport_map, key, se_nacl,
+ GFP_ATOMIC);
}
- sess->s_id = s_id;
+ sess->d_id = s_id;
nacl->nport_id = key;
}
.find_sess_by_s_id = tcm_qla2xxx_find_sess_by_s_id,
.find_sess_by_loop_id = tcm_qla2xxx_find_sess_by_loop_id,
.clear_nacl_from_fcport_map = tcm_qla2xxx_clear_nacl_from_fcport_map,
+ .put_sess = tcm_qla2xxx_put_sess,
.shutdown_sess = tcm_qla2xxx_shutdown_sess,
};
(struct tcm_qla2xxx_lport *)base_vha->vha_tgt.target_lport_ptr;
struct fc_vport_identifiers vport_id;
- if (!qla_tgt_mode_enabled(base_vha)) {
+ if (qla_ini_mode_enabled(base_vha)) {
pr_err("qla2xxx base_vha not enabled for target mode\n");
return -EPERM;
}
p = strchr(tmp, '@');
if (!p) {
- pr_err("Unable to locate NPIV '@' seperator\n");
+ pr_err("Unable to locate NPIV '@' separator\n");
return ERR_PTR(-EINVAL);
}
*p++ = '\0';
u64 nport_wwnn;
/* ASCII formatted WWPN for FC Initiator Nport */
char nport_name[TCM_QLA2XXX_NAMELEN];
- /* Pointer to qla_tgt_sess */
- struct qla_tgt_sess *qla_tgt_sess;
+ /* Pointer to fc_port */
+ struct fc_port *fc_port;
/* Pointer to TCM FC nexus */
struct se_session *nport_nexus;
};
bool scsi_normalize_sense(const u8 *sense_buffer, int sb_len,
struct scsi_sense_hdr *sshdr)
{
+ memset(sshdr, 0, sizeof(struct scsi_sense_hdr));
+
if (!sense_buffer || !sb_len)
return false;
- memset(sshdr, 0, sizeof(struct scsi_sense_hdr));
-
sshdr->response_code = (sense_buffer[0] & 0x7f);
if (!scsi_sense_valid(sshdr))
}
EXPORT_SYMBOL_GPL(scsi_unregister_device_handler);
-static struct scsi_device *get_sdev_from_queue(struct request_queue *q)
-{
- struct scsi_device *sdev;
- unsigned long flags;
-
- spin_lock_irqsave(q->queue_lock, flags);
- sdev = q->queuedata;
- if (!sdev || !get_device(&sdev->sdev_gendev))
- sdev = NULL;
- spin_unlock_irqrestore(q->queue_lock, flags);
-
- return sdev;
-}
-
/*
* scsi_dh_activate - activate the path associated with the scsi_device
* corresponding to the given request queue.
struct scsi_device *sdev;
int err = SCSI_DH_NOSYS;
- sdev = get_sdev_from_queue(q);
+ sdev = scsi_device_from_queue(q);
if (!sdev) {
if (fn)
fn(data, err);
struct scsi_device *sdev;
int err = -SCSI_DH_NOSYS;
- sdev = get_sdev_from_queue(q);
+ sdev = scsi_device_from_queue(q);
if (!sdev)
return err;
struct scsi_device_handler *scsi_dh;
int err = 0;
- sdev = get_sdev_from_queue(q);
+ sdev = scsi_device_from_queue(q);
if (!sdev)
return -ENODEV;
struct scsi_device *sdev;
const char *handler_name = NULL;
- sdev = get_sdev_from_queue(q);
+ sdev = scsi_device_from_queue(q);
if (!sdev)
return NULL;
int scsi_ioctl(struct scsi_device *sdev, int cmd, void __user *arg)
{
char scsi_cmd[MAX_COMMAND_SIZE];
+ struct scsi_sense_hdr sense_hdr;
/* Check for deprecated ioctls ... all the ioctls which don't
* follow the new unique numbering scheme are deprecated */
return scsi_set_medium_removal(sdev, SCSI_REMOVAL_ALLOW);
case SCSI_IOCTL_TEST_UNIT_READY:
return scsi_test_unit_ready(sdev, IOCTL_NORMAL_TIMEOUT,
- NORMAL_RETRIES, NULL);
+ NORMAL_RETRIES, &sense_hdr);
case SCSI_IOCTL_START_UNIT:
scsi_cmd[0] = START_STOP;
scsi_cmd[1] = 0;
__scsi_queue_insert(cmd, reason, 1);
}
-static int __scsi_execute(struct scsi_device *sdev, const unsigned char *cmd,
+
+/**
+ * scsi_execute - insert request and wait for the result
+ * @sdev: scsi device
+ * @cmd: scsi command
+ * @data_direction: data direction
+ * @buffer: data buffer
+ * @bufflen: len of buffer
+ * @sense: optional sense buffer
+ * @sshdr: optional decoded sense header
+ * @timeout: request timeout in seconds
+ * @retries: number of times to retry request
+ * @flags: flags for ->cmd_flags
+ * @rq_flags: flags for ->rq_flags
+ * @resid: optional residual length
+ *
+ * returns the req->errors value which is the scsi_cmnd result
+ * field.
+ */
+int scsi_execute(struct scsi_device *sdev, const unsigned char *cmd,
int data_direction, void *buffer, unsigned bufflen,
- unsigned char *sense, int timeout, int retries, u64 flags,
- req_flags_t rq_flags, int *resid)
+ unsigned char *sense, struct scsi_sense_hdr *sshdr,
+ int timeout, int retries, u64 flags, req_flags_t rq_flags,
+ int *resid)
{
struct request *req;
struct scsi_request *rq;
*resid = rq->resid_len;
if (sense && rq->sense_len)
memcpy(sense, rq->sense, SCSI_SENSE_BUFFERSIZE);
+ if (sshdr)
+ scsi_normalize_sense(rq->sense, rq->sense_len, sshdr);
ret = req->errors;
out:
blk_put_request(req);
return ret;
}
-
-/**
- * scsi_execute - insert request and wait for the result
- * @sdev: scsi device
- * @cmd: scsi command
- * @data_direction: data direction
- * @buffer: data buffer
- * @bufflen: len of buffer
- * @sense: optional sense buffer
- * @timeout: request timeout in seconds
- * @retries: number of times to retry request
- * @flags: or into request flags;
- * @resid: optional residual length
- *
- * returns the req->errors value which is the scsi_cmnd result
- * field.
- */
-int scsi_execute(struct scsi_device *sdev, const unsigned char *cmd,
- int data_direction, void *buffer, unsigned bufflen,
- unsigned char *sense, int timeout, int retries, u64 flags,
- int *resid)
-{
- return __scsi_execute(sdev, cmd, data_direction, buffer, bufflen, sense,
- timeout, retries, flags, 0, resid);
-}
EXPORT_SYMBOL(scsi_execute);
-int scsi_execute_req_flags(struct scsi_device *sdev, const unsigned char *cmd,
- int data_direction, void *buffer, unsigned bufflen,
- struct scsi_sense_hdr *sshdr, int timeout, int retries,
- int *resid, u64 flags, req_flags_t rq_flags)
-{
- char *sense = NULL;
- int result;
-
- if (sshdr) {
- sense = kzalloc(SCSI_SENSE_BUFFERSIZE, GFP_NOIO);
- if (!sense)
- return DRIVER_ERROR << 24;
- }
- result = __scsi_execute(sdev, cmd, data_direction, buffer, bufflen,
- sense, timeout, retries, flags, rq_flags, resid);
- if (sshdr)
- scsi_normalize_sense(sense, SCSI_SENSE_BUFFERSIZE, sshdr);
-
- kfree(sense);
- return result;
-}
-EXPORT_SYMBOL(scsi_execute_req_flags);
-
/*
* Function: scsi_init_cmd_errh()
*
blk_mq_free_tag_set(&shost->tag_set);
}
+/**
+ * scsi_device_from_queue - return sdev associated with a request_queue
+ * @q: The request queue to return the sdev from
+ *
+ * Return the sdev associated with a request queue or NULL if the
+ * request_queue does not reference a SCSI device.
+ */
+struct scsi_device *scsi_device_from_queue(struct request_queue *q)
+{
+ struct scsi_device *sdev = NULL;
+
+ if (q->mq_ops) {
+ if (q->mq_ops == &scsi_mq_ops)
+ sdev = q->queuedata;
+ } else if (q->request_fn == scsi_request_fn)
+ sdev = q->queuedata;
+ if (!sdev || !get_device(&sdev->sdev_gendev))
+ sdev = NULL;
+
+ return sdev;
+}
+EXPORT_SYMBOL_GPL(scsi_device_from_queue);
+
/*
* Function: scsi_block_requests()
*
* @sdev: scsi device to change the state of.
* @timeout: command timeout
* @retries: number of retries before failing
- * @sshdr_external: Optional pointer to struct scsi_sense_hdr for
- * returning sense. Make sure that this is cleared before passing
- * in.
+ * @sshdr: outpout pointer for decoded sense information.
*
* Returns zero if unsuccessful or an error if TUR failed. For
* removable media, UNIT_ATTENTION sets ->changed flag.
**/
int
scsi_test_unit_ready(struct scsi_device *sdev, int timeout, int retries,
- struct scsi_sense_hdr *sshdr_external)
+ struct scsi_sense_hdr *sshdr)
{
char cmd[] = {
TEST_UNIT_READY, 0, 0, 0, 0, 0,
};
- struct scsi_sense_hdr *sshdr;
int result;
- if (!sshdr_external)
- sshdr = kzalloc(sizeof(*sshdr), GFP_KERNEL);
- else
- sshdr = sshdr_external;
-
/* try to eat the UNIT_ATTENTION if there are enough retries */
do {
result = scsi_execute_req(sdev, cmd, DMA_NONE, NULL, 0, sshdr,
} while (scsi_sense_valid(sshdr) &&
sshdr->sense_key == UNIT_ATTENTION && --retries);
- if (!sshdr_external)
- kfree(sshdr);
return result;
}
EXPORT_SYMBOL(scsi_test_unit_ready);
{
int i, result;
unsigned char sense[SCSI_SENSE_BUFFERSIZE];
+ struct scsi_sense_hdr sshdr_tmp;
+
+ if (!sshdr)
+ sshdr = &sshdr_tmp;
for(i = 0; i < DV_RETRIES; i++) {
- result = scsi_execute(sdev, cmd, dir, buffer, bufflen,
- sense, DV_TIMEOUT, /* retries */ 1,
+ result = scsi_execute(sdev, cmd, dir, buffer, bufflen, sense,
+ sshdr, DV_TIMEOUT, /* retries */ 1,
REQ_FAILFAST_DEV |
REQ_FAILFAST_TRANSPORT |
REQ_FAILFAST_DRIVER,
- NULL);
- if (driver_byte(result) & DRIVER_SENSE) {
- struct scsi_sense_hdr sshdr_tmp;
- if (!sshdr)
- sshdr = &sshdr_tmp;
-
- if (scsi_normalize_sense(sense, SCSI_SENSE_BUFFERSIZE,
- sshdr)
- && sshdr->sense_key == UNIT_ATTENTION)
- continue;
- }
- break;
+ 0, NULL);
+ if (!(driver_byte(result) & DRIVER_SENSE) ||
+ sshdr->sense_key != UNIT_ATTENTION)
+ break;
}
return result;
}
{
struct scsi_disk *sdkp = scsi_disk_get(disk);
struct scsi_device *sdp;
- struct scsi_sense_hdr *sshdr = NULL;
int retval;
if (!sdkp)
* by sd_spinup_disk() from sd_revalidate_disk(), which happens whenever
* sd_revalidate() is called.
*/
- retval = -ENODEV;
-
if (scsi_block_when_processing_errors(sdp)) {
- sshdr = kzalloc(sizeof(*sshdr), GFP_KERNEL);
+ struct scsi_sense_hdr sshdr = { 0, };
+
retval = scsi_test_unit_ready(sdp, SD_TIMEOUT, SD_MAX_RETRIES,
- sshdr);
- }
+ &sshdr);
- /* failed to execute TUR, assume media not present */
- if (host_byte(retval)) {
- set_media_not_present(sdkp);
- goto out;
- }
+ /* failed to execute TUR, assume media not present */
+ if (host_byte(retval)) {
+ set_media_not_present(sdkp);
+ goto out;
+ }
- if (media_not_present(sdkp, sshdr))
- goto out;
+ if (media_not_present(sdkp, &sshdr))
+ goto out;
+ }
/*
* For removable scsi disk we have to recognise the presence
* Medium present state has changed in either direction.
* Device has indicated UNIT_ATTENTION.
*/
- kfree(sshdr);
retval = sdp->changed ? DISK_EVENT_MEDIA_CHANGE : 0;
sdp->changed = 0;
scsi_disk_put(sdkp);
* Leave the rest of the command zero to indicate
* flush everything.
*/
- res = scsi_execute_req_flags(sdp, cmd, DMA_NONE, NULL, 0,
- &sshdr, timeout, SD_MAX_RETRIES,
- NULL, 0, RQF_PM);
+ res = scsi_execute(sdp, cmd, DMA_NONE, NULL, 0, NULL, &sshdr,
+ timeout, SD_MAX_RETRIES, 0, RQF_PM, NULL);
if (res == 0)
break;
}
put_device(&sdkp->dev);
}
-struct sd_devt {
- int idx;
- struct disk_devt disk_devt;
-};
-
-void sd_devt_release(struct disk_devt *disk_devt)
-{
- struct sd_devt *sd_devt = container_of(disk_devt, struct sd_devt,
- disk_devt);
-
- spin_lock(&sd_index_lock);
- ida_remove(&sd_index_ida, sd_devt->idx);
- spin_unlock(&sd_index_lock);
-
- kfree(sd_devt);
-}
-
/**
* sd_probe - called during driver initialization and whenever a
* new scsi device is attached to the system. It is called once
static int sd_probe(struct device *dev)
{
struct scsi_device *sdp = to_scsi_device(dev);
- struct sd_devt *sd_devt;
struct scsi_disk *sdkp;
struct gendisk *gd;
int index;
if (!sdkp)
goto out;
- sd_devt = kzalloc(sizeof(*sd_devt), GFP_KERNEL);
- if (!sd_devt)
- goto out_free;
-
gd = alloc_disk(SD_MINORS);
if (!gd)
- goto out_free_devt;
+ goto out_free;
do {
if (!ida_pre_get(&sd_index_ida, GFP_KERNEL))
goto out_put;
}
- atomic_set(&sd_devt->disk_devt.count, 1);
- sd_devt->disk_devt.release = sd_devt_release;
- sd_devt->idx = index;
- gd->disk_devt = &sd_devt->disk_devt;
-
error = sd_format_disk_name("sd", index, gd->disk_name, DISK_NAME_LEN);
if (error) {
sdev_printk(KERN_WARNING, sdp, "SCSI disk (sd) name length exceeded.\n");
return 0;
out_free_index:
- put_disk_devt(&sd_devt->disk_devt);
- sd_devt = NULL;
+ spin_lock(&sd_index_lock);
+ ida_remove(&sd_index_ida, index);
+ spin_unlock(&sd_index_lock);
out_put:
put_disk(gd);
out_free:
kfree(sdkp);
- out_free_devt:
- kfree(sd_devt);
out:
scsi_autopm_put_device(sdp);
return error;
struct scsi_disk *sdkp = to_scsi_disk(dev);
struct gendisk *disk = sdkp->disk;
- put_disk_devt(disk->disk_devt);
+ spin_lock(&sd_index_lock);
+ ida_remove(&sd_index_ida, sdkp->index);
+ spin_unlock(&sd_index_lock);
+
disk->private_data = NULL;
put_disk(disk);
put_device(&sdkp->device->sdev_gendev);
if (!scsi_device_online(sdp))
return -ENODEV;
- res = scsi_execute_req_flags(sdp, cmd, DMA_NONE, NULL, 0, &sshdr,
- SD_TIMEOUT, SD_MAX_RETRIES, NULL, 0, RQF_PM);
+ res = scsi_execute(sdp, cmd, DMA_NONE, NULL, 0, NULL, &sshdr,
+ SD_TIMEOUT, SD_MAX_RETRIES, 0, RQF_PM, NULL);
if (res) {
sd_print_result(sdkp, "Start/Stop Unit failed", res);
if (driver_byte(res) & DRIVER_SENSE)
size_t buffer_length;
time64_t local_time;
unsigned int year;
- struct timeval time;
- struct rtc_time tm;
+ struct tm tm;
buffer_length = sizeof(*buffer);
put_unaligned_le16(sizeof(buffer->time),
&buffer->time_length);
- do_gettimeofday(&time);
- local_time = time.tv_sec - (sys_tz.tz_minuteswest * 60);
- rtc_time64_to_tm(local_time, &tm);
+ local_time = ktime_get_real_seconds();
+ time64_to_tm(local_time, -sys_tz.tz_minuteswest * 60, &tm);
year = tm.tm_year + 1900;
buffer->time[0] = bin2bcd(tm.tm_hour);
struct scsi_device *SDev;
struct scsi_sense_hdr sshdr;
int result, err = 0, retries = 0;
- struct request_sense *sense = cgc->sense;
SDev = cd->device;
- if (!sense) {
- sense = kmalloc(SCSI_SENSE_BUFFERSIZE, GFP_KERNEL);
- if (!sense) {
- err = -ENOMEM;
- goto out;
- }
- }
-
retry:
if (!scsi_block_when_processing_errors(SDev)) {
err = -ENODEV;
goto out;
}
- memset(sense, 0, sizeof(*sense));
result = scsi_execute(SDev, cgc->cmd, cgc->data_direction,
- cgc->buffer, cgc->buflen, (char *)sense,
- cgc->timeout, IOCTL_RETRIES, 0, NULL);
-
- scsi_normalize_sense((char *)sense, sizeof(*sense), &sshdr);
+ cgc->buffer, cgc->buflen,
+ (unsigned char *)cgc->sense, &sshdr,
+ cgc->timeout, IOCTL_RETRIES, 0, 0, NULL);
/* Minimal error checking. Ignore cases we know about, and report the rest. */
if (driver_byte(result) != 0) {
/* Wake up a process waiting for device */
out:
- if (!cgc->sense)
- kfree(sense);
cgc->stat = err;
return err;
}
#include <linux/fs.h>
#include <linux/kernel.h>
-#include <linux/sched.h>
+#include <linux/sched/signal.h>
#include <linux/mm.h>
#include <linux/init.h>
#include <linux/string.h>
return false;
}
- /*
- * Not performing check for each individual select_major
- * mappings of select_minor, since there is no harm in
- * configuring a non-existent select_minor
- */
- if (host->testbus.select_minor > 0xFF) {
- dev_err(host->hba->dev,
- "%s: 0x%05X is not a legal testbus option\n",
- __func__, host->testbus.select_minor);
- return false;
- }
-
return true;
}
goto out;
}
- ret = scsi_execute_req_flags(sdp, cmd, DMA_FROM_DEVICE, buffer,
- UFSHCD_REQ_SENSE_SIZE, NULL,
- msecs_to_jiffies(1000), 3, NULL, 0, RQF_PM);
+ ret = scsi_execute(sdp, cmd, DMA_FROM_DEVICE, buffer,
+ UFSHCD_REQ_SENSE_SIZE, NULL, NULL,
+ msecs_to_jiffies(1000), 3, 0, RQF_PM, NULL);
if (ret)
pr_err("%s: failed with err %d\n", __func__, ret);
* callbacks hence set the RQF_PM flag so that it doesn't resume the
* already suspended childs.
*/
- ret = scsi_execute_req_flags(sdp, cmd, DMA_NONE, NULL, 0, &sshdr,
- START_STOP_TIMEOUT, 0, NULL, 0, RQF_PM);
+ ret = scsi_execute(sdp, cmd, DMA_NONE, NULL, 0, NULL, &sshdr,
+ START_STOP_TIMEOUT, 0, 0, RQF_PM, NULL);
if (ret) {
sdev_printk(KERN_WARNING, sdp,
"START_STOP failed for power mode: %d, result %x\n",
#include <linux/module.h>
#include <linux/slab.h>
#include <linux/mempool.h>
+#include <linux/interrupt.h>
#include <linux/virtio.h>
#include <linux/virtio_ids.h>
#include <linux/virtio_config.h>
#include <scsi/scsi_cmnd.h>
#include <scsi/scsi_tcq.h>
#include <linux/seqlock.h>
+#include <linux/blk-mq-virtio.h>
#define VIRTIO_SCSI_MEMPOOL_SZ 64
#define VIRTIO_SCSI_EVENT_LEN 8
bool affinity_hint_set;
struct hlist_node node;
- struct hlist_node node_dead;
/* Protected by event_vq lock */
bool stop_events;
struct virtio_scsi_vq req_vqs[];
};
-static enum cpuhp_state virtioscsi_online;
static struct kmem_cache *virtscsi_cmd_cache;
static mempool_t *virtscsi_cmd_pool;
kfree(tgt);
}
+static int virtscsi_map_queues(struct Scsi_Host *shost)
+{
+ struct virtio_scsi *vscsi = shost_priv(shost);
+
+ return blk_mq_virtio_map_queues(&shost->tag_set, vscsi->vdev, 2);
+}
+
static struct scsi_host_template virtscsi_host_template_single = {
.module = THIS_MODULE,
.name = "Virtio SCSI HBA",
.use_clustering = ENABLE_CLUSTERING,
.target_alloc = virtscsi_target_alloc,
.target_destroy = virtscsi_target_destroy,
+ .map_queues = virtscsi_map_queues,
.track_queue_depth = 1,
};
virtio_cwrite(vdev, struct virtio_scsi_config, fld, &__val); \
} while(0)
-static void __virtscsi_set_affinity(struct virtio_scsi *vscsi, bool affinity)
-{
- int i;
- int cpu;
-
- /* In multiqueue mode, when the number of cpu is equal
- * to the number of request queues, we let the qeueues
- * to be private to one cpu by setting the affinity hint
- * to eliminate the contention.
- */
- if ((vscsi->num_queues == 1 ||
- vscsi->num_queues != num_online_cpus()) && affinity) {
- if (vscsi->affinity_hint_set)
- affinity = false;
- else
- return;
- }
-
- if (affinity) {
- i = 0;
- for_each_online_cpu(cpu) {
- virtqueue_set_affinity(vscsi->req_vqs[i].vq, cpu);
- i++;
- }
-
- vscsi->affinity_hint_set = true;
- } else {
- for (i = 0; i < vscsi->num_queues; i++) {
- if (!vscsi->req_vqs[i].vq)
- continue;
-
- virtqueue_set_affinity(vscsi->req_vqs[i].vq, -1);
- }
-
- vscsi->affinity_hint_set = false;
- }
-}
-
-static void virtscsi_set_affinity(struct virtio_scsi *vscsi, bool affinity)
-{
- get_online_cpus();
- __virtscsi_set_affinity(vscsi, affinity);
- put_online_cpus();
-}
-
-static int virtscsi_cpu_online(unsigned int cpu, struct hlist_node *node)
-{
- struct virtio_scsi *vscsi = hlist_entry_safe(node, struct virtio_scsi,
- node);
- __virtscsi_set_affinity(vscsi, true);
- return 0;
-}
-
-static int virtscsi_cpu_notif_add(struct virtio_scsi *vi)
-{
- int ret;
-
- ret = cpuhp_state_add_instance(virtioscsi_online, &vi->node);
- if (ret)
- return ret;
-
- ret = cpuhp_state_add_instance(CPUHP_VIRT_SCSI_DEAD, &vi->node_dead);
- if (ret)
- cpuhp_state_remove_instance(virtioscsi_online, &vi->node);
- return ret;
-}
-
-static void virtscsi_cpu_notif_remove(struct virtio_scsi *vi)
-{
- cpuhp_state_remove_instance_nocalls(virtioscsi_online, &vi->node);
- cpuhp_state_remove_instance_nocalls(CPUHP_VIRT_SCSI_DEAD,
- &vi->node_dead);
-}
-
static void virtscsi_init_vq(struct virtio_scsi_vq *virtscsi_vq,
struct virtqueue *vq)
{
static void virtscsi_remove_vqs(struct virtio_device *vdev)
{
- struct Scsi_Host *sh = virtio_scsi_host(vdev);
- struct virtio_scsi *vscsi = shost_priv(sh);
-
- virtscsi_set_affinity(vscsi, false);
-
/* Stop all the virtqueues. */
vdev->config->reset(vdev);
-
vdev->config->del_vqs(vdev);
}
vq_callback_t **callbacks;
const char **names;
struct virtqueue **vqs;
+ struct irq_affinity desc = { .pre_vectors = 2 };
num_vqs = vscsi->num_queues + VIRTIO_SCSI_VQ_BASE;
vqs = kmalloc(num_vqs * sizeof(struct virtqueue *), GFP_KERNEL);
}
/* Discover virtqueues and write information to configuration. */
- err = vdev->config->find_vqs(vdev, num_vqs, vqs, callbacks, names);
+ err = vdev->config->find_vqs(vdev, num_vqs, vqs, callbacks, names,
+ &desc);
if (err)
goto out;
if (err)
goto virtscsi_init_failed;
- err = virtscsi_cpu_notif_add(vscsi);
- if (err)
- goto scsi_add_host_failed;
-
cmd_per_lun = virtscsi_config_get(vdev, cmd_per_lun) ?: 1;
shost->cmd_per_lun = min_t(u32, cmd_per_lun, shost->can_queue);
shost->max_sectors = virtscsi_config_get(vdev, max_sectors) ?: 0xFFFF;
virtscsi_cancel_event_work(vscsi);
scsi_remove_host(shost);
-
- virtscsi_cpu_notif_remove(vscsi);
-
virtscsi_remove_vqs(vdev);
scsi_host_put(shost);
}
#ifdef CONFIG_PM_SLEEP
static int virtscsi_freeze(struct virtio_device *vdev)
{
- struct Scsi_Host *sh = virtio_scsi_host(vdev);
- struct virtio_scsi *vscsi = shost_priv(sh);
-
- virtscsi_cpu_notif_remove(vscsi);
virtscsi_remove_vqs(vdev);
return 0;
}
if (err)
return err;
- err = virtscsi_cpu_notif_add(vscsi);
- if (err) {
- vdev->config->del_vqs(vdev);
- return err;
- }
virtio_device_ready(vdev);
if (virtio_has_feature(vdev, VIRTIO_SCSI_F_HOTPLUG))
pr_err("mempool_create() for virtscsi_cmd_pool failed\n");
goto error;
}
- ret = cpuhp_setup_state_multi(CPUHP_AP_ONLINE_DYN,
- "scsi/virtio:online",
- virtscsi_cpu_online, NULL);
- if (ret < 0)
- goto error;
- virtioscsi_online = ret;
- ret = cpuhp_setup_state_multi(CPUHP_VIRT_SCSI_DEAD, "scsi/virtio:dead",
- NULL, virtscsi_cpu_online);
- if (ret)
- goto error;
ret = register_virtio_driver(&virtio_scsi_driver);
if (ret < 0)
goto error;
kmem_cache_destroy(virtscsi_cmd_cache);
virtscsi_cmd_cache = NULL;
}
- if (virtioscsi_online)
- cpuhp_remove_multi_state(virtioscsi_online);
- cpuhp_remove_multi_state(CPUHP_VIRT_SCSI_DEAD);
return ret;
}
static void __exit fini(void)
{
unregister_virtio_driver(&virtio_scsi_driver);
- cpuhp_remove_multi_state(virtioscsi_online);
- cpuhp_remove_multi_state(CPUHP_VIRT_SCSI_DEAD);
mempool_destroy(virtscsi_cmd_pool);
kmem_cache_destroy(virtscsi_cmd_cache);
}
#include <linux/module.h>
#include <linux/interrupt.h>
#include <linux/kthread.h>
+#include <linux/sched/signal.h>
#include <linux/vmalloc.h>
#include <linux/platform_device.h>
#include <linux/of.h>
#include <linux/pm_domain.h>
#include <linux/export.h>
#include <linux/sched/rt.h>
+#include <uapi/linux/sched/types.h>
#include <linux/delay.h>
#include <linux/kthread.h>
#include <linux/ioport.h>
#include <linux/debugfs.h>
#include <linux/dma-buf.h>
#include <linux/idr.h>
+#include <linux/sched/task.h>
#include "ion.h"
#include "ion_priv.h"
#include <linux/mm.h>
#include <linux/rtmutex.h>
#include <linux/sched.h>
+#include <uapi/linux/sched/types.h>
#include <linux/scatterlist.h>
#include <linux/vmalloc.h>
#include "ion.h"
#include <linux/kernel.h>
#include <linux/mm.h>
#include <linux/oom.h>
-#include <linux/sched.h>
+#include <linux/sched/signal.h>
#include <linux/swap.h>
#include <linux/rcupdate.h>
#include <linux/profile.h>
#include <linux/module.h>
#include <linux/errno.h>
#include <linux/kernel.h>
-#include <linux/sched.h>
+#include <linux/sched/signal.h>
#include <linux/fcntl.h>
#include <linux/delay.h>
#include <linux/mm.h>
*/
#include <linux/kernel.h>
-#include <linux/sched.h> /* For jiffies, task states */
+#include <linux/sched/signal.h> /* For jiffies, task states, etc. */
#include <linux/interrupt.h> /* For tasklet and interrupt structs/defines */
#include <linux/module.h>
#include <linux/ctype.h>
#include <linux/tty.h>
-#include <linux/sched.h>
+#include <linux/sched/signal.h>
#include "dgnc_utils.h"
/*
int *paddr_cells,
int *mc_addr_cells,
int *mc_size_cells,
- const __be32 **ranges_start,
- u8 *num_ranges)
+ const __be32 **ranges_start)
{
const __be32 *prop;
int range_tuple_cell_count;
dev_warn(dev,
"missing or empty ranges property for device tree node '%s'\n",
mc_node->name);
-
- *num_ranges = 0;
return 0;
}
return -EINVAL;
}
- *num_ranges = ranges_len / tuple_len;
- return 0;
+ return ranges_len / tuple_len;
}
static int get_mc_addr_translation_ranges(struct device *dev,
**ranges,
u8 *num_ranges)
{
- int error;
+ int ret;
int paddr_cells;
int mc_addr_cells;
int mc_size_cells;
const __be32 *ranges_start;
const __be32 *cell;
- error = parse_mc_ranges(dev,
+ ret = parse_mc_ranges(dev,
&paddr_cells,
&mc_addr_cells,
&mc_size_cells,
- &ranges_start,
- num_ranges);
- if (error < 0)
- return error;
+ &ranges_start);
+ if (ret < 0)
+ return ret;
- if (!(*num_ranges)) {
+ *num_ranges = ret;
+ if (!ret) {
/*
* Missing or empty ranges property ("ranges;") for the
* 'fsl,qoriq-mc' node. In this case, identity mapping
#include <linux/kernel.h>
#include <linux/errno.h>
#include <linux/module.h>
-#include <linux/sched.h>
+#include <linux/sched/signal.h>
#include <linux/wait.h>
#include <linux/slab.h>
#include <linux/uaccess.h>
#include <linux/module.h>
#include <linux/kernel.h>
#include <linux/fs_struct.h>
-#include <linux/sched.h>
+#include <linux/sched/signal.h>
#include "../../../include/linux/libcfs/libcfs.h"
svc->sv_shuttingdown = 0;
svc->sv_cpt_data = cfs_percpt_alloc(lnet_cpt_table(),
- sizeof(*svc->sv_cpt_data));
+ sizeof(**svc->sv_cpt_data));
if (!svc->sv_cpt_data)
return -ENOMEM;
#ifdef __KERNEL__
# include <linux/quota.h>
+# include <linux/sched/signal.h>
# include <linux/string.h> /* snprintf() */
# include <linux/version.h>
#else /* !__KERNEL__ */
#include <linux/fs_struct.h>
#include <linux/namei.h>
+#include <linux/cred.h>
#include "lustre_patchless_compat.h"
* @{
*/
-#include <linux/sched.h>
+#include <linux/sched/signal.h>
#include <linux/signal.h>
#include <linux/types.h>
#include "../../include/linux/libcfs/libcfs.h"
#define _OBD_SUPPORT
#include <linux/slab.h>
+#include <linux/sched/signal.h>
+
#include "../../include/linux/libcfs/libcfs.h"
#include "lustre_compat.h"
#include "lprocfs_status.h"
return rc;
}
-int ll_getattr(struct vfsmount *mnt, struct dentry *de, struct kstat *stat)
+int ll_getattr(const struct path *path, struct kstat *stat,
+ u32 request_mask, unsigned int flags)
{
- struct inode *inode = d_inode(de);
+ struct inode *inode = d_inode(path->dentry);
struct ll_sb_info *sbi = ll_i2sbi(inode);
struct ll_inode_info *lli = ll_i2info(inode);
int res;
- res = ll_inode_revalidate(de, MDS_INODELOCK_UPDATE |
- MDS_INODELOCK_LOOKUP);
+ res = ll_inode_revalidate(path->dentry,
+ MDS_INODELOCK_UPDATE | MDS_INODELOCK_LOOKUP);
ll_stats_ops_tally(sbi, LPROC_LL_GETATTR, 1);
if (res)
int ll_file_release(struct inode *inode, struct file *file);
int ll_release_openhandle(struct inode *, struct lookup_intent *);
int ll_md_real_close(struct inode *inode, fmode_t fmode);
-int ll_getattr(struct vfsmount *mnt, struct dentry *de, struct kstat *stat);
+int ll_getattr(const struct path *path, struct kstat *stat,
+ u32 request_mask, unsigned int flags);
struct posix_acl *ll_get_acl(struct inode *inode, int type);
int ll_migrate(struct inode *parent, struct file *file, int mdtidx,
const char *name, int namelen);
#include "../../include/linux/libcfs/libcfs.h"
#include <linux/crypto.h>
+#include <linux/cred.h>
#include <linux/key.h>
+#include <linux/sched/task.h>
#include "../include/obd.h"
#include "../include/obd_class.h"
#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
#include <linux/module.h>
-#include <linux/sched.h>
+#include <linux/sched/signal.h>
#include <linux/errno.h>
#include <linux/signal.h>
#include <linux/fs.h>
#include <linux/module.h>
#include <linux/kmod.h>
#include <linux/kernel.h>
-#include <linux/sched.h>
+#include <linux/sched/signal.h>
#include <linux/fs.h>
#include <linux/poll.h>
#include <linux/string.h>
/* get context */
msg_context = get_msg_context(instance);
- if (msg_context == NULL)
- return -ENOMEM;
+ if (!msg_context) {
+ ret = -ENOMEM;
+ goto unlock;
+ }
/* store bulk message context for when data arrives */
msg_context->u.bulk.instance = instance;
vchi_service_release(instance->handle);
+unlock:
mutex_unlock(&instance->bulk_mutex);
return ret;
if (likely((port < TOTAL_NUMBER_OF_PORTS) &&
cvm_oct_device[port])) {
struct net_device *dev = cvm_oct_device[port];
- struct octeon_ethernet *priv = netdev_priv(dev);
/*
* Only accept packets for devices that are
#include <linux/io.h>
#include <linux/mutex.h>
#include <linux/sem.h>
-#include <linux/sched.h>
+#include <linux/sched/signal.h>
#include <linux/etherdevice.h>
#include <linux/wireless.h>
#include <net/iw_handler.h>
#include <linux/interrupt.h>
#include <linux/semaphore.h>
-#include <linux/sched.h>
+#include <linux/sched/signal.h>
#include <linux/sem.h>
#include <linux/netdevice.h>
#include <linux/etherdevice.h>
#include <linux/kernel.h>
#include <linux/errno.h>
#include <linux/slab.h>
+#include <linux/sched/signal.h>
#include <linux/module.h>
#include <linux/kref.h>
#include <linux/netdevice.h>
#include <linux/unistd.h>
#include <linux/miscdevice.h> /* for misc_register, and SYNTH_MINOR */
#include <linux/poll.h> /* for poll_wait() */
-#include <linux/sched.h> /* schedule(), signal_pending(), TASK_INTERRUPTIBLE */
+#include <linux/sched/signal.h> /* schedule(), signal_pending(), TASK_INTERRUPTIBLE */
#include "spk_priv.h"
#include "speakup.h"
config BCM2835_VCHIQ
tristate "Videocore VCHIQ"
depends on HAS_DMA
+ depends on OF
depends on RASPBERRYPI_FIRMWARE || (COMPILE_TEST && !RASPBERRYPI_FIRMWARE)
default y
help
if (err < 0)
return err;
- (void)of_property_read_u32(dev->of_node, "cache-line-size",
+ err = of_property_read_u32(dev->of_node, "cache-line-size",
&g_cache_line_size);
+
+ if (err) {
+ dev_err(dev, "Missing cache-line-size property\n");
+ return -ENODEV;
+ }
+
g_fragments_size = 2 * g_cache_line_size;
/* Allocate space for the channels in coherent memory */
#include <linux/kernel.h>
#include <linux/module.h>
+#include <linux/sched/signal.h>
#include <linux/types.h>
#include <linux/errno.h>
#include <linux/cdev.h>
#include <linux/types.h>
#include <linux/interrupt.h>
#include <linux/random.h>
-#include <linux/sched.h>
+#include <linux/sched/signal.h>
#include <linux/ctype.h>
#include <linux/uaccess.h>
#include <linux/time.h> /* for time_t */
goto out;
csk->mtu = ndev->mtu;
csk->tx_chan = cxgb4_port_chan(ndev);
- csk->smac_idx = (cxgb4_port_viid(ndev) & 0x7F) << 1;
+ csk->smac_idx = cxgb4_tp_smt_idx(cdev->lldi.adapter_type,
+ cxgb4_port_viid(ndev));
step = cdev->lldi.ntxq /
cdev->lldi.nchan;
csk->txq_idx = cxgb4_port_idx(ndev) * step;
port_id = cxgb4_port_idx(ndev);
csk->mtu = dst_mtu(dst);
csk->tx_chan = cxgb4_port_chan(ndev);
- csk->smac_idx = (cxgb4_port_viid(ndev) & 0x7F) << 1;
+ csk->smac_idx = cxgb4_tp_smt_idx(cdev->lldi.adapter_type,
+ cxgb4_port_viid(ndev));
step = cdev->lldi.ntxq /
cdev->lldi.nports;
csk->txq_idx = (port_id * step) +
struct sk_buff *skb;
const struct tcphdr *tcph;
struct cpl_t5_pass_accept_rpl *rpl5;
+ struct cxgb4_lld_info *lldi = &csk->com.cdev->lldi;
unsigned int len = roundup(sizeof(*rpl5), 16);
unsigned int mtu_idx;
u64 opt0;
opt2 = RX_CHANNEL_V(0) |
RSS_QUEUE_VALID_F | RSS_QUEUE_V(csk->rss_qid);
+ if (!is_t5(lldi->adapter_type))
+ opt2 |= RX_FC_DISABLE_F;
+
if (req->tcpopt.tstamp)
opt2 |= TSTAMPS_EN_F;
if (req->tcpopt.sack)
opt2 |= WND_SCALE_EN_F;
hlen = ntohl(req->hdr_len);
- tcph = (const void *)(req + 1) + ETH_HDR_LEN_G(hlen) +
- IP_HDR_LEN_G(hlen);
+
+ if (is_t5(lldi->adapter_type))
+ tcph = (struct tcphdr *)((u8 *)(req + 1) +
+ ETH_HDR_LEN_G(hlen) + IP_HDR_LEN_G(hlen));
+ else
+ tcph = (struct tcphdr *)((u8 *)(req + 1) +
+ T6_ETH_HDR_LEN_G(hlen) + T6_IP_HDR_LEN_G(hlen));
if (tcph->ece && tcph->cwr)
opt2 |= CCTRL_ECN_V(1);
}
while (skb) {
- credit += skb->csum;
+ credit += (__force u32)skb->csum;
skb = cxgbit_skcb_tx_wr_next(skb);
}
while (credits) {
struct sk_buff *p = cxgbit_sock_peek_wr(csk);
+ const u32 csum = (__force u32)p->csum;
if (unlikely(!p)) {
pr_err("csk 0x%p,%u, cr %u,%u+%u, empty.\n",
break;
}
- if (unlikely(credits < p->csum)) {
+ if (unlikely(credits < csum)) {
pr_warn("csk 0x%p,%u, cr %u,%u+%u, < %u.\n",
csk, csk->tid,
credits, csk->wr_cred, csk->wr_una_cred,
- p->csum);
- p->csum -= credits;
+ csum);
+ p->csum = (__force __wsum)(csum - credits);
break;
}
cxgbit_sock_dequeue_wr(csk);
- credits -= p->csum;
+ credits -= csum;
kfree_skb(p);
}
PDUCBF_RX_DATA = (1 << 1), /* received pdu payload */
PDUCBF_RX_STATUS = (1 << 2), /* received ddp status */
PDUCBF_RX_DATA_DDPD = (1 << 3), /* pdu payload ddp'd */
- PDUCBF_RX_HCRC_ERR = (1 << 4), /* header digest error */
- PDUCBF_RX_DCRC_ERR = (1 << 5), /* data digest error */
+ PDUCBF_RX_DDP_CMP = (1 << 4), /* ddp completion */
+ PDUCBF_RX_HCRC_ERR = (1 << 5), /* header digest error */
+ PDUCBF_RX_DCRC_ERR = (1 << 6), /* data digest error */
};
struct cxgbit_lro_pdu_cb {
}
static void
-cxgbit_proc_ddp_status(unsigned int tid, struct cpl_rx_data_ddp *cpl,
- struct cxgbit_lro_pdu_cb *pdu_cb)
+cxgbit_process_ddpvld(struct cxgbit_sock *csk, struct cxgbit_lro_pdu_cb *pdu_cb,
+ u32 ddpvld)
{
- unsigned int status = ntohl(cpl->ddpvld);
- pdu_cb->flags |= PDUCBF_RX_STATUS;
- pdu_cb->ddigest = ntohl(cpl->ulp_crc);
- pdu_cb->pdulen = ntohs(cpl->len);
-
- if (status & (1 << CPL_RX_ISCSI_DDP_STATUS_HCRC_SHIFT)) {
- pr_info("tid 0x%x, status 0x%x, hcrc bad.\n", tid, status);
+ if (ddpvld & (1 << CPL_RX_ISCSI_DDP_STATUS_HCRC_SHIFT)) {
+ pr_info("tid 0x%x, status 0x%x, hcrc bad.\n", csk->tid, ddpvld);
pdu_cb->flags |= PDUCBF_RX_HCRC_ERR;
}
- if (status & (1 << CPL_RX_ISCSI_DDP_STATUS_DCRC_SHIFT)) {
- pr_info("tid 0x%x, status 0x%x, dcrc bad.\n", tid, status);
+ if (ddpvld & (1 << CPL_RX_ISCSI_DDP_STATUS_DCRC_SHIFT)) {
+ pr_info("tid 0x%x, status 0x%x, dcrc bad.\n", csk->tid, ddpvld);
pdu_cb->flags |= PDUCBF_RX_DCRC_ERR;
}
- if (status & (1 << CPL_RX_ISCSI_DDP_STATUS_PAD_SHIFT))
- pr_info("tid 0x%x, status 0x%x, pad bad.\n", tid, status);
+ if (ddpvld & (1 << CPL_RX_ISCSI_DDP_STATUS_PAD_SHIFT))
+ pr_info("tid 0x%x, status 0x%x, pad bad.\n", csk->tid, ddpvld);
- if ((status & (1 << CPL_RX_ISCSI_DDP_STATUS_DDP_SHIFT)) &&
+ if ((ddpvld & (1 << CPL_RX_ISCSI_DDP_STATUS_DDP_SHIFT)) &&
(!(pdu_cb->flags & PDUCBF_RX_DATA))) {
pdu_cb->flags |= PDUCBF_RX_DATA_DDPD;
}
lro_cb->pdu_idx);
struct cpl_rx_iscsi_ddp *cpl = (struct cpl_rx_iscsi_ddp *)(rsp + 1);
- cxgbit_proc_ddp_status(lro_cb->csk->tid, cpl, pdu_cb);
+ cxgbit_process_ddpvld(lro_cb->csk, pdu_cb, be32_to_cpu(cpl->ddpvld));
+
+ pdu_cb->flags |= PDUCBF_RX_STATUS;
+ pdu_cb->ddigest = ntohl(cpl->ulp_crc);
+ pdu_cb->pdulen = ntohs(cpl->len);
if (pdu_cb->flags & PDUCBF_RX_HDR)
pdu_cb->complete = true;
- lro_cb->complete = true;
lro_cb->pdu_totallen += pdu_cb->pdulen;
+ lro_cb->complete = true;
lro_cb->pdu_idx++;
}
cxgbit_skcb_flags(skb) = 0;
lro_cb->complete = false;
- } else {
+ } else if (op == CPL_ISCSI_DATA) {
struct cpl_iscsi_data *cpl = (struct cpl_iscsi_data *)gl->va;
offset = sizeof(struct cpl_iscsi_data);
pdu_cb->doffset = lro_cb->offset;
pdu_cb->nr_dfrags = gl->nfrags;
pdu_cb->dfrag_idx = skb_shinfo(skb)->nr_frags;
+ lro_cb->complete = false;
+ } else {
+ struct cpl_rx_iscsi_cmp *cpl;
+
+ cpl = (struct cpl_rx_iscsi_cmp *)gl->va;
+ offset = sizeof(struct cpl_rx_iscsi_cmp);
+ pdu_cb->flags |= (PDUCBF_RX_HDR | PDUCBF_RX_STATUS);
+ len = be16_to_cpu(cpl->len);
+ pdu_cb->hdr = gl->va + offset;
+ pdu_cb->hlen = len;
+ pdu_cb->hfrag_idx = skb_shinfo(skb)->nr_frags;
+ pdu_cb->ddigest = be32_to_cpu(cpl->ulp_crc);
+ pdu_cb->pdulen = ntohs(cpl->len);
+
+ if (unlikely(gl->nfrags > 1))
+ cxgbit_skcb_flags(skb) = 0;
+
+ cxgbit_process_ddpvld(lro_cb->csk, pdu_cb,
+ be32_to_cpu(cpl->ddpvld));
+
+ if (pdu_cb->flags & PDUCBF_RX_DATA_DDPD) {
+ pdu_cb->flags |= PDUCBF_RX_DDP_CMP;
+ pdu_cb->complete = true;
+ } else if (pdu_cb->flags & PDUCBF_RX_DATA) {
+ pdu_cb->complete = true;
+ }
+
+ lro_cb->pdu_totallen += pdu_cb->hlen + pdu_cb->dlen;
+ lro_cb->complete = true;
+ lro_cb->pdu_idx++;
}
cxgbit_copy_frags(skb, gl, offset);
switch (op) {
case CPL_ISCSI_HDR:
case CPL_ISCSI_DATA:
+ case CPL_RX_ISCSI_CMP:
case CPL_RX_ISCSI_DDP:
case CPL_FW4_ACK:
lro_flush = false;
if (unlikely(op != *(u8 *)gl->va)) {
pr_info("? FL 0x%p,RSS%#llx,FL %#llx,len %u.\n",
gl->va, be64_to_cpu(*rsp),
- be64_to_cpu(*(u64 *)gl->va),
+ get_unaligned_be64(gl->va),
gl->tot_len);
return 0;
}
- if (op == CPL_ISCSI_HDR || op == CPL_ISCSI_DATA) {
+ if ((op == CPL_ISCSI_HDR) || (op == CPL_ISCSI_DATA) ||
+ (op == CPL_RX_ISCSI_CMP)) {
if (!cxgbit_lro_receive(csk, op, rsp, gl, lro_mgr,
napi))
return 0;
#include <linux/workqueue.h>
#include <linux/kthread.h>
+#include <linux/sched/signal.h>
+
#include <asm/unaligned.h>
#include <net/tcp.h>
#include <target/target_core_base.h>
u32 len, u32 credits, u32 compl)
{
struct fw_ofld_tx_data_wr *req;
+ const struct cxgb4_lld_info *lldi = &csk->com.cdev->lldi;
u32 submode = cxgbit_skcb_submode(skb);
u32 wr_ulp_mode = 0;
u32 hdr_size = sizeof(*req);
u32 opcode = FW_OFLD_TX_DATA_WR;
u32 immlen = 0;
- u32 force = TX_FORCE_V(!submode);
+ u32 force = is_t5(lldi->adapter_type) ? TX_FORCE_V(!submode) :
+ T6_TX_FORCE_F;
if (cxgbit_skcb_flags(skb) & SKCBF_TX_ISO) {
opcode = FW_ISCSI_TX_DATA_WR;
}
__skb_unlink(skb, &csk->txq);
set_wr_txq(skb, CPL_PRIORITY_DATA, csk->txq_idx);
- skb->csum = credits_needed + flowclen16;
+ skb->csum = (__force __wsum)(credits_needed + flowclen16);
csk->wr_cred -= credits_needed;
csk->wr_una_cred += credits_needed;
u32 max_npdu, max_iso_npdu;
if (conn->login->leading_connection) {
- param = iscsi_find_param_from_key(DATASEQUENCEINORDER,
- conn->param_list);
- if (!param) {
- pr_err("param not found key %s\n", DATASEQUENCEINORDER);
- return -1;
- }
-
- if (strcmp(param->value, YES))
- return 0;
-
- param = iscsi_find_param_from_key(DATAPDUINORDER,
- conn->param_list);
- if (!param) {
- pr_err("param not found key %s\n", DATAPDUINORDER);
- return -1;
- }
-
- if (strcmp(param->value, YES))
- return 0;
-
param = iscsi_find_param_from_key(MAXBURSTLENGTH,
conn->param_list);
if (!param) {
if (kstrtou32(param->value, 0, &mbl) < 0)
return -1;
} else {
- if (!conn->sess->sess_ops->DataSequenceInOrder)
- return 0;
- if (!conn->sess->sess_ops->DataPDUInOrder)
- return 0;
-
mbl = conn->sess->sess_ops->MaxBurstLength;
}
return 0;
}
+/*
+ * cxgbit_seq_pdu_inorder()
+ * @csk: pointer to cxgbit socket structure
+ *
+ * This function checks whether data sequence and data
+ * pdu are in order.
+ *
+ * Return: returns -1 on error, 0 if data sequence and
+ * data pdu are in order, 1 if data sequence or data pdu
+ * is not in order.
+ */
+static int cxgbit_seq_pdu_inorder(struct cxgbit_sock *csk)
+{
+ struct iscsi_conn *conn = csk->conn;
+ struct iscsi_param *param;
+
+ if (conn->login->leading_connection) {
+ param = iscsi_find_param_from_key(DATASEQUENCEINORDER,
+ conn->param_list);
+ if (!param) {
+ pr_err("param not found key %s\n", DATASEQUENCEINORDER);
+ return -1;
+ }
+
+ if (strcmp(param->value, YES))
+ return 1;
+
+ param = iscsi_find_param_from_key(DATAPDUINORDER,
+ conn->param_list);
+ if (!param) {
+ pr_err("param not found key %s\n", DATAPDUINORDER);
+ return -1;
+ }
+
+ if (strcmp(param->value, YES))
+ return 1;
+
+ } else {
+ if (!conn->sess->sess_ops->DataSequenceInOrder)
+ return 1;
+ if (!conn->sess->sess_ops->DataPDUInOrder)
+ return 1;
+ }
+
+ return 0;
+}
+
static int cxgbit_set_params(struct iscsi_conn *conn)
{
struct cxgbit_sock *csk = conn->context;
}
if (!erl) {
+ int ret;
+
+ ret = cxgbit_seq_pdu_inorder(csk);
+ if (ret < 0) {
+ return -1;
+ } else if (ret > 0) {
+ if (is_t5(cdev->lldi.adapter_type))
+ goto enable_ddp;
+ else
+ goto enable_digest;
+ }
+
if (test_bit(CDEV_ISO_ENABLE, &cdev->flags)) {
if (cxgbit_set_iso_npdu(csk))
return -1;
}
+enable_ddp:
if (test_bit(CDEV_DDP_ENABLE, &cdev->flags)) {
if (cxgbit_setup_conn_pgidx(csk,
ppm->tformat.pgsz_idx_dflt))
}
}
+enable_digest:
if (cxgbit_set_digest(csk))
return -1;
int rc, sg_nents, sg_off;
bool dcrc_err = false;
- rc = iscsit_check_dataout_hdr(conn, (unsigned char *)hdr, &cmd);
- if (rc < 0)
- return rc;
- else if (!cmd)
- return 0;
+ if (pdu_cb->flags & PDUCBF_RX_DDP_CMP) {
+ u32 offset = be32_to_cpu(hdr->offset);
+ u32 ddp_data_len;
+ u32 payload_length = ntoh24(hdr->dlength);
+ bool success = false;
+
+ cmd = iscsit_find_cmd_from_itt_or_dump(conn, hdr->itt, 0);
+ if (!cmd)
+ return 0;
+
+ ddp_data_len = offset - cmd->write_data_done;
+ atomic_long_add(ddp_data_len, &conn->sess->rx_data_octets);
+
+ cmd->write_data_done = offset;
+ cmd->next_burst_len = ddp_data_len;
+ cmd->data_sn = be32_to_cpu(hdr->datasn);
+
+ rc = __iscsit_check_dataout_hdr(conn, (unsigned char *)hdr,
+ cmd, payload_length, &success);
+ if (rc < 0)
+ return rc;
+ else if (!success)
+ return 0;
+ } else {
+ rc = iscsit_check_dataout_hdr(conn, (unsigned char *)hdr, &cmd);
+ if (rc < 0)
+ return rc;
+ else if (!cmd)
+ return 0;
+ }
if (pdu_cb->flags & PDUCBF_RX_DCRC_ERR) {
pr_err("ITT: 0x%08x, Offset: %u, Length: %u,"
for (i = 0; i < ssi->nr_frags; i++)
put_page(skb_frag_page(&ssi->frags[i]));
ssi->nr_frags = 0;
+ skb->data_len = 0;
+ skb->truesize -= skb->len;
+ skb->len = 0;
}
static void
unsigned int len = 0;
if (pdu_cb->flags & PDUCBF_RX_HDR) {
- hpdu_cb->flags = pdu_cb->flags;
+ u8 hfrag_idx = hssi->nr_frags;
+
+ hpdu_cb->flags |= pdu_cb->flags;
hpdu_cb->seq = pdu_cb->seq;
hpdu_cb->hdr = pdu_cb->hdr;
hpdu_cb->hlen = pdu_cb->hlen;
- memcpy(&hssi->frags[0], &ssi->frags[pdu_cb->hfrag_idx],
+ memcpy(&hssi->frags[hfrag_idx], &ssi->frags[pdu_cb->hfrag_idx],
sizeof(skb_frag_t));
- get_page(skb_frag_page(&hssi->frags[0]));
- hssi->nr_frags = 1;
- hpdu_cb->frags = 1;
- hpdu_cb->hfrag_idx = 0;
+ get_page(skb_frag_page(&hssi->frags[hfrag_idx]));
+ hssi->nr_frags++;
+ hpdu_cb->frags++;
+ hpdu_cb->hfrag_idx = hfrag_idx;
- len = hssi->frags[0].size;
- hskb->len = len;
- hskb->data_len = len;
- hskb->truesize = len;
+ len = hssi->frags[hfrag_idx].size;
+ hskb->len += len;
+ hskb->data_len += len;
+ hskb->truesize += len;
}
if (pdu_cb->flags & PDUCBF_RX_DATA) {
- u8 hfrag_idx = 1, i;
+ u8 dfrag_idx = hssi->nr_frags, i;
hpdu_cb->flags |= pdu_cb->flags;
+ hpdu_cb->dfrag_idx = dfrag_idx;
len = 0;
- for (i = 0; i < pdu_cb->nr_dfrags; hfrag_idx++, i++) {
- memcpy(&hssi->frags[hfrag_idx],
+ for (i = 0; i < pdu_cb->nr_dfrags; dfrag_idx++, i++) {
+ memcpy(&hssi->frags[dfrag_idx],
&ssi->frags[pdu_cb->dfrag_idx + i],
sizeof(skb_frag_t));
- get_page(skb_frag_page(&hssi->frags[hfrag_idx]));
+ get_page(skb_frag_page(&hssi->frags[dfrag_idx]));
- len += hssi->frags[hfrag_idx].size;
+ len += hssi->frags[dfrag_idx].size;
hssi->nr_frags++;
hpdu_cb->frags++;
hpdu_cb->dlen = pdu_cb->dlen;
hpdu_cb->doffset = hpdu_cb->hlen;
hpdu_cb->nr_dfrags = pdu_cb->nr_dfrags;
- hpdu_cb->dfrag_idx = 1;
hskb->len += len;
hskb->data_len += len;
hskb->truesize += len;
static int cxgbit_rx_skb(struct cxgbit_sock *csk, struct sk_buff *skb)
{
+ struct cxgb4_lld_info *lldi = &csk->com.cdev->lldi;
int ret = -1;
- if (likely(cxgbit_skcb_flags(skb) & SKCBF_RX_LRO))
- ret = cxgbit_rx_lro_skb(csk, skb);
+ if (likely(cxgbit_skcb_flags(skb) & SKCBF_RX_LRO)) {
+ if (is_t5(lldi->adapter_type))
+ ret = cxgbit_rx_lro_skb(csk, skb);
+ else
+ ret = cxgbit_process_lro_skb(csk, skb);
+ }
__kfree_skb(skb);
return ret;
#include <linux/vmalloc.h>
#include <linux/idr.h>
#include <linux/delay.h>
+#include <linux/sched/signal.h>
#include <asm/unaligned.h>
#include <net/ipv6.h>
#include <scsi/scsi_proto.h>
}
int
-iscsit_check_dataout_hdr(struct iscsi_conn *conn, unsigned char *buf,
- struct iscsi_cmd **out_cmd)
+__iscsit_check_dataout_hdr(struct iscsi_conn *conn, void *buf,
+ struct iscsi_cmd *cmd, u32 payload_length,
+ bool *success)
{
- struct iscsi_data *hdr = (struct iscsi_data *)buf;
- struct iscsi_cmd *cmd = NULL;
+ struct iscsi_data *hdr = buf;
struct se_cmd *se_cmd;
- u32 payload_length = ntoh24(hdr->dlength);
int rc;
- if (!payload_length) {
- pr_warn("DataOUT payload is ZERO, ignoring.\n");
- return 0;
- }
-
/* iSCSI write */
atomic_long_add(payload_length, &conn->sess->rx_data_octets);
- if (payload_length > conn->conn_ops->MaxXmitDataSegmentLength) {
- pr_err("DataSegmentLength: %u is greater than"
- " MaxXmitDataSegmentLength: %u\n", payload_length,
- conn->conn_ops->MaxXmitDataSegmentLength);
- return iscsit_add_reject(conn, ISCSI_REASON_PROTOCOL_ERROR,
- buf);
- }
-
- cmd = iscsit_find_cmd_from_itt_or_dump(conn, hdr->itt,
- payload_length);
- if (!cmd)
- return 0;
-
pr_debug("Got DataOut ITT: 0x%08x, TTT: 0x%08x,"
" DataSN: 0x%08x, Offset: %u, Length: %u, CID: %hu\n",
hdr->itt, hdr->ttt, hdr->datasn, ntohl(hdr->offset),
}
}
/*
- * Preform DataSN, DataSequenceInOrder, DataPDUInOrder, and
+ * Perform DataSN, DataSequenceInOrder, DataPDUInOrder, and
* within-command recovery checks before receiving the payload.
*/
rc = iscsit_check_pre_dataout(cmd, buf);
return 0;
else if (rc == DATAOUT_CANNOT_RECOVER)
return -1;
-
- *out_cmd = cmd;
+ *success = true;
return 0;
}
+EXPORT_SYMBOL(__iscsit_check_dataout_hdr);
+
+int
+iscsit_check_dataout_hdr(struct iscsi_conn *conn, void *buf,
+ struct iscsi_cmd **out_cmd)
+{
+ struct iscsi_data *hdr = buf;
+ struct iscsi_cmd *cmd;
+ u32 payload_length = ntoh24(hdr->dlength);
+ int rc;
+ bool success = false;
+
+ if (!payload_length) {
+ pr_warn_ratelimited("DataOUT payload is ZERO, ignoring.\n");
+ return 0;
+ }
+
+ if (payload_length > conn->conn_ops->MaxXmitDataSegmentLength) {
+ pr_err_ratelimited("DataSegmentLength: %u is greater than"
+ " MaxXmitDataSegmentLength: %u\n", payload_length,
+ conn->conn_ops->MaxXmitDataSegmentLength);
+ return iscsit_add_reject(conn, ISCSI_REASON_PROTOCOL_ERROR, buf);
+ }
+
+ cmd = iscsit_find_cmd_from_itt_or_dump(conn, hdr->itt, payload_length);
+ if (!cmd)
+ return 0;
+
+ rc = __iscsit_check_dataout_hdr(conn, buf, cmd, payload_length, &success);
+
+ if (success)
+ *out_cmd = cmd;
+
+ return rc;
+}
EXPORT_SYMBOL(iscsit_check_dataout_hdr);
static int
return ret;
}
+static enum tcm_tmreq_table iscsit_convert_tmf(u8 iscsi_tmf)
+{
+ switch (iscsi_tmf) {
+ case ISCSI_TM_FUNC_ABORT_TASK:
+ return TMR_ABORT_TASK;
+ case ISCSI_TM_FUNC_ABORT_TASK_SET:
+ return TMR_ABORT_TASK_SET;
+ case ISCSI_TM_FUNC_CLEAR_ACA:
+ return TMR_CLEAR_ACA;
+ case ISCSI_TM_FUNC_CLEAR_TASK_SET:
+ return TMR_CLEAR_TASK_SET;
+ case ISCSI_TM_FUNC_LOGICAL_UNIT_RESET:
+ return TMR_LUN_RESET;
+ case ISCSI_TM_FUNC_TARGET_WARM_RESET:
+ return TMR_TARGET_WARM_RESET;
+ case ISCSI_TM_FUNC_TARGET_COLD_RESET:
+ return TMR_TARGET_COLD_RESET;
+ default:
+ return TMR_UNKNOWN;
+ }
+}
+
int
iscsit_handle_task_mgt_cmd(struct iscsi_conn *conn, struct iscsi_cmd *cmd,
unsigned char *buf)
struct iscsi_tm *hdr;
int out_of_order_cmdsn = 0, ret;
bool sess_ref = false;
- u8 function;
+ u8 function, tcm_function = TMR_UNKNOWN;
hdr = (struct iscsi_tm *) buf;
hdr->flags &= ~ISCSI_FLAG_CMD_FINAL;
* LIO-Target $FABRIC_MOD
*/
if (function != ISCSI_TM_FUNC_TASK_REASSIGN) {
-
- u8 tcm_function;
- int ret;
-
transport_init_se_cmd(&cmd->se_cmd, &iscsi_ops,
conn->sess->se_sess, 0, DMA_NONE,
TCM_SIMPLE_TAG, cmd->sense_buffer + 2);
target_get_sess_cmd(&cmd->se_cmd, true);
sess_ref = true;
-
- switch (function) {
- case ISCSI_TM_FUNC_ABORT_TASK:
- tcm_function = TMR_ABORT_TASK;
- break;
- case ISCSI_TM_FUNC_ABORT_TASK_SET:
- tcm_function = TMR_ABORT_TASK_SET;
- break;
- case ISCSI_TM_FUNC_CLEAR_ACA:
- tcm_function = TMR_CLEAR_ACA;
- break;
- case ISCSI_TM_FUNC_CLEAR_TASK_SET:
- tcm_function = TMR_CLEAR_TASK_SET;
- break;
- case ISCSI_TM_FUNC_LOGICAL_UNIT_RESET:
- tcm_function = TMR_LUN_RESET;
- break;
- case ISCSI_TM_FUNC_TARGET_WARM_RESET:
- tcm_function = TMR_TARGET_WARM_RESET;
- break;
- case ISCSI_TM_FUNC_TARGET_COLD_RESET:
- tcm_function = TMR_TARGET_COLD_RESET;
- break;
- default:
+ tcm_function = iscsit_convert_tmf(function);
+ if (tcm_function == TMR_UNKNOWN) {
pr_err("Unknown iSCSI TMR Function:"
" 0x%02x\n", function);
return iscsit_add_reject_cmd(cmd,
ISCSI_REASON_BOOKMARK_NO_RESOURCES, buf);
}
-
- ret = core_tmr_alloc_req(&cmd->se_cmd, cmd->tmr_req,
- tcm_function, GFP_KERNEL);
- if (ret < 0)
- return iscsit_add_reject_cmd(cmd,
+ }
+ ret = core_tmr_alloc_req(&cmd->se_cmd, cmd->tmr_req, tcm_function,
+ GFP_KERNEL);
+ if (ret < 0)
+ return iscsit_add_reject_cmd(cmd,
ISCSI_REASON_BOOKMARK_NO_RESOURCES, buf);
- cmd->tmr_req->se_tmr_req = cmd->se_cmd.se_tmr_req;
- }
+ cmd->tmr_req->se_tmr_req = cmd->se_cmd.se_tmr_req;
cmd->iscsi_opcode = ISCSI_OP_SCSI_TMFUNC;
cmd->i_state = ISTATE_SEND_TASKMGTRSP;
/*
* During Connection recovery drop unacknowledged out of order
* commands for this connection, and prepare the other commands
- * for realligence.
+ * for reallegiance.
*
* During normal operation clear the out of order commands (but
* do not free the struct iscsi_ooo_cmdsn's) and release all
*/
if (atomic_read(&conn->connection_recovery)) {
iscsit_discard_unacknowledged_ooo_cmdsns_for_conn(conn);
- iscsit_prepare_cmds_for_realligance(conn);
+ iscsit_prepare_cmds_for_reallegiance(conn);
} else {
iscsit_clear_ooo_cmdsns_for_conn(conn);
iscsit_release_commands_from_conn(conn);
* GNU General Public License for more details.
******************************************************************************/
+#include <linux/sched/signal.h>
+
#include <scsi/iscsi_proto.h>
#include <target/target_core_base.h>
#include <target/target_core_fabric.h>
*/
if (cmd->unsolicited_data) {
cmd->seq_start_offset = cmd->write_data_done;
- cmd->seq_end_offset = (cmd->write_data_done +
- ((cmd->se_cmd.data_length >
- conn->sess->sess_ops->FirstBurstLength) ?
- conn->sess->sess_ops->FirstBurstLength : cmd->se_cmd.data_length));
+ cmd->seq_end_offset = min(cmd->se_cmd.data_length,
+ conn->sess->sess_ops->FirstBurstLength);
return;
}
return 0;
}
-int iscsit_prepare_cmds_for_realligance(struct iscsi_conn *conn)
+int iscsit_prepare_cmds_for_reallegiance(struct iscsi_conn *conn)
{
u32 cmd_count = 0;
struct iscsi_cmd *cmd, *cmd_tmp;
if ((cmd->iscsi_opcode != ISCSI_OP_SCSI_CMD) &&
(cmd->iscsi_opcode != ISCSI_OP_NOOP_OUT)) {
- pr_debug("Not performing realligence on"
+ pr_debug("Not performing reallegiance on"
" Opcode: 0x%02x, ITT: 0x%08x, CmdSN: 0x%08x,"
" CID: %hu\n", cmd->iscsi_opcode,
cmd->init_task_tag, cmd->cmd_sn, conn->cid);
cmd_count++;
pr_debug("Preparing Opcode: 0x%02x, ITT: 0x%08x,"
" CmdSN: 0x%08x, StatSN: 0x%08x, CID: %hu for"
- " realligence.\n", cmd->iscsi_opcode,
+ " reallegiance.\n", cmd->iscsi_opcode,
cmd->init_task_tag, cmd->cmd_sn, cmd->stat_sn,
conn->cid);
struct iscsi_session *);
extern void iscsit_discard_cr_cmds_by_expstatsn(struct iscsi_conn_recovery *, u32);
extern int iscsit_discard_unacknowledged_ooo_cmdsns_for_conn(struct iscsi_conn *);
-extern int iscsit_prepare_cmds_for_realligance(struct iscsi_conn *);
+extern int iscsit_prepare_cmds_for_reallegiance(struct iscsi_conn *);
extern int iscsit_connection_recovery_transport_reset(struct iscsi_conn *);
#endif /*** ISCSI_TARGET_ERL2_H ***/
#include <linux/module.h>
#include <linux/string.h>
#include <linux/kthread.h>
+#include <linux/sched/signal.h>
#include <linux/idr.h>
#include <linux/tcp.h> /* TCP_NODELAY */
#include <net/ipv6.h> /* ipv6_addr_v4mapped() */
return 0;
pr_debug("%s iSCSI Session SID %u is still active for %s,"
- " preforming session reinstatement.\n", (sessiontype) ?
+ " performing session reinstatement.\n", (sessiontype) ?
"Discovery" : "Normal", sess->sid,
sess->sess_ops->InitiatorName);
#include <linux/ctype.h>
#include <linux/kthread.h>
#include <linux/slab.h>
+#include <linux/sched/signal.h>
#include <net/sock.h>
#include <scsi/iscsi_proto.h>
#include <target/target_core_base.h>
{
int ret;
- if (conn->sock) {
- struct sock *sk = conn->sock->sk;
+ if (conn->sock) {
+ struct sock *sk = conn->sock->sk;
- write_lock_bh(&sk->sk_callback_lock);
- set_bit(LOGIN_FLAGS_READY, &conn->login_flags);
- write_unlock_bh(&sk->sk_callback_lock);
- }
+ write_lock_bh(&sk->sk_callback_lock);
+ set_bit(LOGIN_FLAGS_READY, &conn->login_flags);
+ write_unlock_bh(&sk->sk_callback_lock);
+ }
- ret = iscsi_target_do_login(conn, login);
- if (ret < 0) {
+ ret = iscsi_target_do_login(conn, login);
+ if (ret < 0) {
cancel_delayed_work_sync(&conn->login_work);
cancel_delayed_work_sync(&conn->login_cleanup_work);
iscsi_target_restore_sock_callbacks(conn);
break;
default:
pr_err("Illegal iSCSI Opcode 0x%02x during"
- " command realligence\n", cmd->iscsi_opcode);
+ " command reallegiance\n", cmd->iscsi_opcode);
return -1;
}
if (ret != 0)
return ret;
- pr_debug("Completed connection realligence for Opcode: 0x%02x,"
+ pr_debug("Completed connection reallegiance for Opcode: 0x%02x,"
" ITT: 0x%08x to CID: %hu.\n", cmd->iscsi_opcode,
cmd->init_task_tag, conn->cid);
return NULL;
}
+EXPORT_SYMBOL(iscsit_find_cmd_from_itt_or_dump);
struct iscsi_cmd *iscsit_find_cmd_from_ttt(
struct iscsi_conn *conn,
return total_rx;
}
-static int iscsit_do_tx_data(
- struct iscsi_conn *conn,
- struct iscsi_data_count *count)
-{
- int ret, iov_len;
- struct kvec *iov_p;
- struct msghdr msg;
-
- if (!conn || !conn->sock || !conn->conn_ops)
- return -1;
-
- if (count->data_length <= 0) {
- pr_err("Data length is: %d\n", count->data_length);
- return -1;
- }
-
- memset(&msg, 0, sizeof(struct msghdr));
-
- iov_p = count->iov;
- iov_len = count->iov_count;
-
- ret = kernel_sendmsg(conn->sock, &msg, iov_p, iov_len,
- count->data_length);
- if (ret != count->data_length) {
- pr_err("Unexpected ret: %d send data %d\n",
- ret, count->data_length);
- return -EPIPE;
- }
- pr_debug("ret: %d, sent data: %d\n", ret, count->data_length);
-
- return ret;
-}
-
int rx_data(
struct iscsi_conn *conn,
struct kvec *iov,
int iov_count,
int data)
{
- struct iscsi_data_count c;
+ struct msghdr msg;
+ int total_tx = 0;
if (!conn || !conn->sock || !conn->conn_ops)
return -1;
- memset(&c, 0, sizeof(struct iscsi_data_count));
- c.iov = iov;
- c.iov_count = iov_count;
- c.data_length = data;
- c.type = ISCSI_TX_DATA;
+ if (data <= 0) {
+ pr_err("Data length is: %d\n", data);
+ return -1;
+ }
- return iscsit_do_tx_data(conn, &c);
-}
+ memset(&msg, 0, sizeof(struct msghdr));
-static bool sockaddr_equal(struct sockaddr_storage *x, struct sockaddr_storage *y)
-{
- switch (x->ss_family) {
- case AF_INET: {
- struct sockaddr_in *sinx = (struct sockaddr_in *)x;
- struct sockaddr_in *siny = (struct sockaddr_in *)y;
- if (sinx->sin_addr.s_addr != siny->sin_addr.s_addr)
- return false;
- if (sinx->sin_port != siny->sin_port)
- return false;
- break;
- }
- case AF_INET6: {
- struct sockaddr_in6 *sinx = (struct sockaddr_in6 *)x;
- struct sockaddr_in6 *siny = (struct sockaddr_in6 *)y;
- if (!ipv6_addr_equal(&sinx->sin6_addr, &siny->sin6_addr))
- return false;
- if (sinx->sin6_port != siny->sin6_port)
- return false;
- break;
- }
- default:
- return false;
+ iov_iter_kvec(&msg.msg_iter, WRITE | ITER_KVEC,
+ iov, iov_count, data);
+
+ while (msg_data_left(&msg)) {
+ int tx_loop = sock_sendmsg(conn->sock, &msg);
+ if (tx_loop <= 0) {
+ pr_debug("tx_loop: %d total_tx %d\n",
+ tx_loop, total_tx);
+ return tx_loop;
+ }
+ total_tx += tx_loop;
+ pr_debug("tx_loop: %d, total_tx: %d, data: %d\n",
+ tx_loop, total_tx, data);
}
- return true;
+
+ return total_tx;
}
void iscsit_collect_login_stats(
ls = &tiqn->login_stats;
spin_lock(&ls->lock);
- if (sockaddr_equal(&conn->login_sockaddr, &ls->last_intr_fail_sockaddr) &&
- ((get_jiffies_64() - ls->last_fail_time) < 10)) {
- /* We already have the failure info for this login */
- spin_unlock(&ls->lock);
- return;
- }
-
if (status_class == ISCSI_STATUS_CLS_SUCCESS)
ls->accepts++;
else if (status_class == ISCSI_STATUS_CLS_REDIRECT) {
{
struct iscsi_portal_group *tpg;
- if (!conn || !conn->sess)
+ if (!conn)
return NULL;
- tpg = conn->sess->tpg;
+ tpg = conn->tpg;
if (!tpg)
return NULL;
&deve->read_bytes);
se_lun = rcu_dereference(deve->se_lun);
+
+ if (!percpu_ref_tryget_live(&se_lun->lun_ref)) {
+ se_lun = NULL;
+ goto out_unlock;
+ }
+
se_cmd->se_lun = rcu_dereference(deve->se_lun);
se_cmd->pr_res_key = deve->pr_res_key;
se_cmd->orig_fe_lun = unpacked_lun;
se_cmd->se_cmd_flags |= SCF_SE_LUN_CMD;
-
- percpu_ref_get(&se_lun->lun_ref);
se_cmd->lun_ref_active = true;
if ((se_cmd->data_direction == DMA_TO_DEVICE) &&
goto ref_dev;
}
}
+out_unlock:
rcu_read_unlock();
if (!se_lun) {
rcu_read_lock();
deve = target_nacl_find_deve(nacl, unpacked_lun);
if (deve) {
- se_tmr->tmr_lun = rcu_dereference(deve->se_lun);
se_cmd->se_lun = rcu_dereference(deve->se_lun);
se_lun = rcu_dereference(deve->se_lun);
se_cmd->pr_res_key = deve->pr_res_key;
xcopy_lun = &dev->xcopy_lun;
rcu_assign_pointer(xcopy_lun->lun_se_dev, dev);
init_completion(&xcopy_lun->lun_ref_comp);
+ init_completion(&xcopy_lun->lun_shutdown_comp);
INIT_LIST_HEAD(&xcopy_lun->lun_deve_list);
INIT_LIST_HEAD(&xcopy_lun->lun_dev_link);
mutex_init(&xcopy_lun->lun_tg_pt_md_mutex);
spin_lock_irq(&cmd->t_state_lock);
cmd->t_state = TRANSPORT_PROCESSING;
- cmd->transport_state |= CMD_T_ACTIVE|CMD_T_BUSY|CMD_T_SENT;
+ cmd->transport_state |= CMD_T_ACTIVE | CMD_T_SENT;
spin_unlock_irq(&cmd->t_state_lock);
__target_execute_cmd(cmd, false);
atomic_long_read(&to_stat_tgt_dev(item)->num_resets));
}
+static ssize_t target_stat_tgt_aborts_complete_show(struct config_item *item,
+ char *page)
+{
+ return snprintf(page, PAGE_SIZE, "%lu\n",
+ atomic_long_read(&to_stat_tgt_dev(item)->aborts_complete));
+}
+
+static ssize_t target_stat_tgt_aborts_no_task_show(struct config_item *item,
+ char *page)
+{
+ return snprintf(page, PAGE_SIZE, "%lu\n",
+ atomic_long_read(&to_stat_tgt_dev(item)->aborts_no_task));
+}
+
CONFIGFS_ATTR_RO(target_stat_tgt_, inst);
CONFIGFS_ATTR_RO(target_stat_tgt_, indx);
CONFIGFS_ATTR_RO(target_stat_tgt_, num_lus);
CONFIGFS_ATTR_RO(target_stat_tgt_, status);
CONFIGFS_ATTR_RO(target_stat_tgt_, non_access_lus);
CONFIGFS_ATTR_RO(target_stat_tgt_, resets);
+CONFIGFS_ATTR_RO(target_stat_tgt_, aborts_complete);
+CONFIGFS_ATTR_RO(target_stat_tgt_, aborts_no_task);
static struct configfs_attribute *target_stat_scsi_tgt_dev_attrs[] = {
&target_stat_tgt_attr_inst,
&target_stat_tgt_attr_status,
&target_stat_tgt_attr_non_access_lus,
&target_stat_tgt_attr_resets,
+ &target_stat_tgt_attr_aborts_complete,
+ &target_stat_tgt_attr_aborts_no_task,
NULL,
};
return ret;
}
+static ssize_t target_stat_transport_proto_id_show(struct config_item *item,
+ char *page)
+{
+ struct se_lun *lun = to_transport_stat(item);
+ struct se_device *dev;
+ struct se_portal_group *tpg = lun->lun_tpg;
+ ssize_t ret = -ENODEV;
+
+ rcu_read_lock();
+ dev = rcu_dereference(lun->lun_se_dev);
+ if (dev)
+ ret = snprintf(page, PAGE_SIZE, "%u\n", tpg->proto_id);
+ rcu_read_unlock();
+ return ret;
+}
+
CONFIGFS_ATTR_RO(target_stat_transport_, inst);
CONFIGFS_ATTR_RO(target_stat_transport_, device);
CONFIGFS_ATTR_RO(target_stat_transport_, indx);
CONFIGFS_ATTR_RO(target_stat_transport_, dev_name);
+CONFIGFS_ATTR_RO(target_stat_transport_, proto_id);
static struct configfs_attribute *target_stat_scsi_transport_attrs[] = {
&target_stat_transport_attr_inst,
&target_stat_transport_attr_device,
&target_stat_transport_attr_indx,
&target_stat_transport_attr_dev_name,
+ &target_stat_transport_attr_proto_id,
NULL,
};
printk("ABORT_TASK: Found referenced %s task_tag: %llu\n",
se_cmd->se_tfo->get_fabric_name(), ref_tag);
- if (!__target_check_io_state(se_cmd, se_sess, 0)) {
- spin_unlock_irqrestore(&se_sess->sess_cmd_lock, flags);
- goto out;
- }
+ if (!__target_check_io_state(se_cmd, se_sess, 0))
+ continue;
+
list_del_init(&se_cmd->se_cmd_list);
spin_unlock_irqrestore(&se_sess->sess_cmd_lock, flags);
printk("ABORT_TASK: Sending TMR_FUNCTION_COMPLETE for"
" ref_tag: %llu\n", ref_tag);
tmr->response = TMR_FUNCTION_COMPLETE;
+ atomic_long_inc(&dev->aborts_complete);
return;
}
spin_unlock_irqrestore(&se_sess->sess_cmd_lock, flags);
-out:
printk("ABORT_TASK: Sending TMR_TASK_DOES_NOT_EXIST for ref_tag: %lld\n",
tmr->ref_task_tag);
tmr->response = TMR_TASK_DOES_NOT_EXIST;
+ atomic_long_inc(&dev->aborts_no_task);
}
static void core_tmr_drain_tmr_list(
* LUN_RESET tmr..
*/
spin_lock_irqsave(&dev->se_tmr_lock, flags);
+ list_del_init(&tmr->tmr_list);
list_for_each_entry_safe(tmr_p, tmr_pp, &dev->dev_tmr_list, tmr_list) {
- /*
- * Allow the received TMR to return with FUNCTION_COMPLETE.
- */
- if (tmr_p == tmr)
- continue;
-
cmd = tmr_p->task_cmd;
if (!cmd) {
pr_err("Unable to locate struct se_cmd for TMR\n");
{
struct se_lun *lun = container_of(ref, struct se_lun, lun_ref);
- complete(&lun->lun_ref_comp);
+ complete(&lun->lun_shutdown_comp);
}
/* Does not change se_wwn->priv. */
lun->lun_link_magic = SE_LUN_LINK_MAGIC;
atomic_set(&lun->lun_acl_count, 0);
init_completion(&lun->lun_ref_comp);
+ init_completion(&lun->lun_shutdown_comp);
INIT_LIST_HEAD(&lun->lun_deve_list);
INIT_LIST_HEAD(&lun->lun_dev_link);
atomic_set(&lun->lun_tg_pt_secondary_offline, 0);
if (!dev)
return;
- if (cmd->transport_state & CMD_T_BUSY)
- return;
-
spin_lock_irqsave(&dev->execute_task_lock, flags);
if (cmd->state_active) {
list_del(&cmd->state_list);
spin_unlock_irqrestore(&dev->execute_task_lock, flags);
}
-static int transport_cmd_check_stop(struct se_cmd *cmd, bool remove_from_lists,
- bool write_pending)
+static int transport_cmd_check_stop_to_fabric(struct se_cmd *cmd)
{
unsigned long flags;
- if (remove_from_lists) {
- target_remove_from_state_list(cmd);
+ target_remove_from_state_list(cmd);
- /*
- * Clear struct se_cmd->se_lun before the handoff to FE.
- */
- cmd->se_lun = NULL;
- }
+ /*
+ * Clear struct se_cmd->se_lun before the handoff to FE.
+ */
+ cmd->se_lun = NULL;
spin_lock_irqsave(&cmd->t_state_lock, flags);
- if (write_pending)
- cmd->t_state = TRANSPORT_WRITE_PENDING;
-
/*
* Determine if frontend context caller is requesting the stopping of
* this command for frontend exceptions.
complete_all(&cmd->t_transport_stop_comp);
return 1;
}
-
cmd->transport_state &= ~CMD_T_ACTIVE;
- if (remove_from_lists) {
- /*
- * Some fabric modules like tcm_loop can release
- * their internally allocated I/O reference now and
- * struct se_cmd now.
- *
- * Fabric modules are expected to return '1' here if the
- * se_cmd being passed is released at this point,
- * or zero if not being released.
- */
- if (cmd->se_tfo->check_stop_free != NULL) {
- spin_unlock_irqrestore(&cmd->t_state_lock, flags);
- return cmd->se_tfo->check_stop_free(cmd);
- }
- }
-
spin_unlock_irqrestore(&cmd->t_state_lock, flags);
- return 0;
-}
-static int transport_cmd_check_stop_to_fabric(struct se_cmd *cmd)
-{
- return transport_cmd_check_stop(cmd, true, false);
+ /*
+ * Some fabric modules like tcm_loop can release their internally
+ * allocated I/O reference and struct se_cmd now.
+ *
+ * Fabric modules are expected to return '1' here if the se_cmd being
+ * passed is released at this point, or zero if not being released.
+ */
+ return cmd->se_tfo->check_stop_free ? cmd->se_tfo->check_stop_free(cmd)
+ : 0;
}
static void transport_lun_remove_cmd(struct se_cmd *cmd)
spin_lock_irqsave(&cmd->t_state_lock, flags);
- cmd->transport_state &= ~CMD_T_BUSY;
if (dev && dev->transport->transport_complete) {
dev->transport->transport_complete(cmd,
init_completion(&cmd->cmd_wait_comp);
spin_lock_init(&cmd->t_state_lock);
kref_init(&cmd->cmd_kref);
- cmd->transport_state = CMD_T_DEV_ACTIVE;
cmd->se_tfo = tfo;
cmd->se_sess = se_sess;
{
int ret = 0, post_ret = 0;
+ if (transport_check_aborted_status(cmd, 1))
+ return;
+
pr_debug("-----[ Storage Engine Exception for cmd: %p ITT: 0x%08llx"
" CDB: 0x%02x\n", cmd, cmd->tag, cmd->t_task_cdb[0]);
pr_debug("-----[ i_state: %d t_state: %d sense_reason: %d\n",
return;
err:
spin_lock_irq(&cmd->t_state_lock);
- cmd->transport_state &= ~(CMD_T_BUSY|CMD_T_SENT);
+ cmd->transport_state &= ~CMD_T_SENT;
spin_unlock_irq(&cmd->t_state_lock);
transport_generic_request_failure(cmd, ret);
sectors, 0, cmd->t_prot_sg, 0);
if (unlikely(cmd->pi_err)) {
spin_lock_irq(&cmd->t_state_lock);
- cmd->transport_state &= ~(CMD_T_BUSY|CMD_T_SENT);
+ cmd->transport_state &= ~CMD_T_SENT;
spin_unlock_irq(&cmd->t_state_lock);
transport_generic_request_failure(cmd, cmd->pi_err);
return -1;
}
cmd->t_state = TRANSPORT_PROCESSING;
- cmd->transport_state |= CMD_T_ACTIVE|CMD_T_BUSY|CMD_T_SENT;
+ cmd->transport_state |= CMD_T_ACTIVE | CMD_T_SENT;
spin_unlock_irq(&cmd->t_state_lock);
if (target_write_prot_action(cmd))
if (target_handle_task_attr(cmd)) {
spin_lock_irq(&cmd->t_state_lock);
- cmd->transport_state &= ~(CMD_T_BUSY | CMD_T_SENT);
+ cmd->transport_state &= ~CMD_T_SENT;
spin_unlock_irq(&cmd->t_state_lock);
return;
}
if (cmd->sam_task_attr == TCM_SIMPLE_TAG) {
atomic_dec_mb(&dev->simple_cmds);
dev->dev_cur_ordered_id++;
- pr_debug("Incremented dev->dev_cur_ordered_id: %u for SIMPLE\n",
- dev->dev_cur_ordered_id);
} else if (cmd->sam_task_attr == TCM_HEAD_TAG) {
dev->dev_cur_ordered_id++;
pr_debug("Incremented dev_cur_ordered_id: %u for HEAD_OF_QUEUE\n",
sense_reason_t
transport_generic_new_cmd(struct se_cmd *cmd)
{
+ unsigned long flags;
int ret = 0;
bool zero_flag = !(cmd->se_cmd_flags & SCF_SCSI_DATA_CDB);
target_execute_cmd(cmd);
return 0;
}
- transport_cmd_check_stop(cmd, false, true);
+
+ spin_lock_irqsave(&cmd->t_state_lock, flags);
+ cmd->t_state = TRANSPORT_WRITE_PENDING;
+ /*
+ * Determine if frontend context caller is requesting the stopping of
+ * this command for frontend exceptions.
+ */
+ if (cmd->transport_state & CMD_T_STOP) {
+ pr_debug("%s:%d CMD_T_STOP for ITT: 0x%08llx\n",
+ __func__, __LINE__, cmd->tag);
+
+ spin_unlock_irqrestore(&cmd->t_state_lock, flags);
+
+ complete_all(&cmd->t_transport_stop_comp);
+ return 0;
+ }
+ cmd->transport_state &= ~CMD_T_ACTIVE;
+ spin_unlock_irqrestore(&cmd->t_state_lock, flags);
ret = cmd->se_tfo->write_pending(cmd);
if (ret == -EAGAIN || ret == -ENOMEM)
unsigned long flags;
bool fabric_stop;
- spin_lock_irqsave(&se_sess->sess_cmd_lock, flags);
+ if (se_sess) {
+ spin_lock_irqsave(&se_sess->sess_cmd_lock, flags);
- spin_lock(&se_cmd->t_state_lock);
- fabric_stop = (se_cmd->transport_state & CMD_T_FABRIC_STOP) &&
- (se_cmd->transport_state & CMD_T_ABORTED);
- spin_unlock(&se_cmd->t_state_lock);
+ spin_lock(&se_cmd->t_state_lock);
+ fabric_stop = (se_cmd->transport_state & CMD_T_FABRIC_STOP) &&
+ (se_cmd->transport_state & CMD_T_ABORTED);
+ spin_unlock(&se_cmd->t_state_lock);
- if (se_cmd->cmd_wait_set || fabric_stop) {
+ if (se_cmd->cmd_wait_set || fabric_stop) {
+ list_del_init(&se_cmd->se_cmd_list);
+ spin_unlock_irqrestore(&se_sess->sess_cmd_lock, flags);
+ target_free_cmd_mem(se_cmd);
+ complete(&se_cmd->cmd_wait_comp);
+ return;
+ }
list_del_init(&se_cmd->se_cmd_list);
spin_unlock_irqrestore(&se_sess->sess_cmd_lock, flags);
- target_free_cmd_mem(se_cmd);
- complete(&se_cmd->cmd_wait_comp);
- return;
}
- list_del_init(&se_cmd->se_cmd_list);
- spin_unlock_irqrestore(&se_sess->sess_cmd_lock, flags);
target_free_cmd_mem(se_cmd);
se_cmd->se_tfo->release_cmd(se_cmd);
}
-/* target_put_sess_cmd - Check for active I/O shutdown via kref_put
- * @se_cmd: command descriptor to drop
+/**
+ * target_put_sess_cmd - decrease the command reference count
+ * @se_cmd: command to drop a reference from
+ *
+ * Returns 1 if and only if this target_put_sess_cmd() call caused the
+ * refcount to drop to zero. Returns zero otherwise.
*/
int target_put_sess_cmd(struct se_cmd *se_cmd)
{
- struct se_session *se_sess = se_cmd->se_sess;
-
- if (!se_sess) {
- target_free_cmd_mem(se_cmd);
- se_cmd->se_tfo->release_cmd(se_cmd);
- return 1;
- }
return kref_put(&se_cmd->cmd_kref, target_release_cmd_kref);
}
EXPORT_SYMBOL(target_put_sess_cmd);
}
EXPORT_SYMBOL(target_wait_for_sess_cmds);
+static void target_lun_confirm(struct percpu_ref *ref)
+{
+ struct se_lun *lun = container_of(ref, struct se_lun, lun_ref);
+
+ complete(&lun->lun_ref_comp);
+}
+
void transport_clear_lun_ref(struct se_lun *lun)
{
- percpu_ref_kill(&lun->lun_ref);
+ /*
+ * Mark the percpu-ref as DEAD, switch to atomic_t mode, drop
+ * the initial reference and schedule confirm kill to be
+ * executed after one full RCU grace period has completed.
+ */
+ percpu_ref_kill_and_confirm(&lun->lun_ref, target_lun_confirm);
+ /*
+ * The first completion waits for percpu_ref_switch_to_atomic_rcu()
+ * to call target_lun_confirm after lun->lun_ref has been marked
+ * as __PERCPU_REF_DEAD on all CPUs, and switches to atomic_t
+ * mode so that percpu_ref_tryget_live() lookup of lun->lun_ref
+ * fails for all new incoming I/O.
+ */
wait_for_completion(&lun->lun_ref_comp);
+ /*
+ * The second completion waits for percpu_ref_put_many() to
+ * invoke ->release() after lun->lun_ref has switched to
+ * atomic_t mode, and lun->lun_ref.count has reached zero.
+ *
+ * At this point all target-core lun->lun_ref references have
+ * been dropped via transport_lun_remove_cmd(), and it's safe
+ * to proceed with the remaining LUN shutdown.
+ */
+ wait_for_completion(&lun->lun_shutdown_comp);
}
static bool
}
/**
- * transport_wait_for_tasks - wait for completion to occur
- * @cmd: command to wait
- *
- * Called from frontend fabric context to wait for storage engine
- * to pause and/or release frontend generated struct se_cmd.
+ * transport_wait_for_tasks - set CMD_T_STOP and wait for t_transport_stop_comp
+ * @cmd: command to wait on
*/
bool transport_wait_for_tasks(struct se_cmd *cmd)
{
static void ft_free_cmd(struct ft_cmd *cmd)
{
struct fc_frame *fp;
- struct fc_lport *lport;
struct ft_sess *sess;
if (!cmd)
return;
sess = cmd->sess;
fp = cmd->req_frame;
- lport = fr_dev(fp);
if (fr_seq(fp))
fc_seq_release(fr_seq(fp));
fc_frame_free(fp);
#include <linux/debugfs.h>
#include <linux/seq_file.h>
#include <linux/sched/rt.h>
+#include <uapi/linux/sched/types.h>
#include <asm/nmi.h>
#include <asm/msr.h>
#include <linux/errno.h>
#include <linux/signal.h>
#include <linux/fcntl.h>
-#include <linux/sched.h>
+#include <linux/sched/signal.h>
#include <linux/interrupt.h>
#include <linux/tty.h>
#include <linux/ctype.h>
#define DEFAULT_TX_BUF_COUNT 3
struct n_hdlc_buf {
- struct n_hdlc_buf *link;
+ struct list_head list_item;
int count;
char buf[1];
};
#define N_HDLC_BUF_SIZE (sizeof(struct n_hdlc_buf) + maxframe)
struct n_hdlc_buf_list {
- struct n_hdlc_buf *head;
- struct n_hdlc_buf *tail;
+ struct list_head list;
int count;
spinlock_t spinlock;
};
* @backup_tty - TTY to use if tty gets closed
* @tbusy - reentrancy flag for tx wakeup code
* @woke_up - FIXME: describe this field
- * @tbuf - currently transmitting tx buffer
* @tx_buf_list - list of pending transmit frame buffers
* @rx_buf_list - list of received frame buffers
* @tx_free_buf_list - list unused transmit frame buffers
struct tty_struct *backup_tty;
int tbusy;
int woke_up;
- struct n_hdlc_buf *tbuf;
struct n_hdlc_buf_list tx_buf_list;
struct n_hdlc_buf_list rx_buf_list;
struct n_hdlc_buf_list tx_free_buf_list;
/*
* HDLC buffer list manipulation functions
*/
+static void n_hdlc_buf_return(struct n_hdlc_buf_list *buf_list,
+ struct n_hdlc_buf *buf);
static void n_hdlc_buf_put(struct n_hdlc_buf_list *list,
struct n_hdlc_buf *buf);
static struct n_hdlc_buf *n_hdlc_buf_get(struct n_hdlc_buf_list *list);
{
struct n_hdlc *n_hdlc = tty2n_hdlc(tty);
struct n_hdlc_buf *buf;
- unsigned long flags;
while ((buf = n_hdlc_buf_get(&n_hdlc->tx_buf_list)))
n_hdlc_buf_put(&n_hdlc->tx_free_buf_list, buf);
- spin_lock_irqsave(&n_hdlc->tx_buf_list.spinlock, flags);
- if (n_hdlc->tbuf) {
- n_hdlc_buf_put(&n_hdlc->tx_free_buf_list, n_hdlc->tbuf);
- n_hdlc->tbuf = NULL;
- }
- spin_unlock_irqrestore(&n_hdlc->tx_buf_list.spinlock, flags);
}
static struct tty_ldisc_ops n_hdlc_ldisc = {
} else
break;
}
- kfree(n_hdlc->tbuf);
kfree(n_hdlc);
} /* end of n_hdlc_release() */
n_hdlc->woke_up = 0;
spin_unlock_irqrestore(&n_hdlc->tx_buf_list.spinlock, flags);
- /* get current transmit buffer or get new transmit */
- /* buffer from list of pending transmit buffers */
-
- tbuf = n_hdlc->tbuf;
- if (!tbuf)
- tbuf = n_hdlc_buf_get(&n_hdlc->tx_buf_list);
-
+ tbuf = n_hdlc_buf_get(&n_hdlc->tx_buf_list);
while (tbuf) {
if (debuglevel >= DEBUG_LEVEL_INFO)
printk("%s(%d)sending frame %p, count=%d\n",
/* rollback was possible and has been done */
if (actual == -ERESTARTSYS) {
- n_hdlc->tbuf = tbuf;
+ n_hdlc_buf_return(&n_hdlc->tx_buf_list, tbuf);
break;
}
/* if transmit error, throw frame away by */
/* free current transmit buffer */
n_hdlc_buf_put(&n_hdlc->tx_free_buf_list, tbuf);
-
- /* this tx buffer is done */
- n_hdlc->tbuf = NULL;
-
+
/* wait up sleeping writers */
wake_up_interruptible(&tty->write_wait);
if (debuglevel >= DEBUG_LEVEL_INFO)
printk("%s(%d)frame %p pending\n",
__FILE__,__LINE__,tbuf);
-
- /* buffer not accepted by driver */
- /* set this buffer as pending buffer */
- n_hdlc->tbuf = tbuf;
+
+ /*
+ * the buffer was not accepted by driver,
+ * return it back into tx queue
+ */
+ n_hdlc_buf_return(&n_hdlc->tx_buf_list, tbuf);
break;
}
}
int error = 0;
int count;
unsigned long flags;
-
+ struct n_hdlc_buf *buf = NULL;
+
if (debuglevel >= DEBUG_LEVEL_INFO)
printk("%s(%d)n_hdlc_tty_ioctl() called %d\n",
__FILE__,__LINE__,cmd);
/* report count of read data available */
/* in next available frame (if any) */
spin_lock_irqsave(&n_hdlc->rx_buf_list.spinlock,flags);
- if (n_hdlc->rx_buf_list.head)
- count = n_hdlc->rx_buf_list.head->count;
+ buf = list_first_entry_or_null(&n_hdlc->rx_buf_list.list,
+ struct n_hdlc_buf, list_item);
+ if (buf)
+ count = buf->count;
else
count = 0;
spin_unlock_irqrestore(&n_hdlc->rx_buf_list.spinlock,flags);
count = tty_chars_in_buffer(tty);
/* add size of next output frame in queue */
spin_lock_irqsave(&n_hdlc->tx_buf_list.spinlock,flags);
- if (n_hdlc->tx_buf_list.head)
- count += n_hdlc->tx_buf_list.head->count;
+ buf = list_first_entry_or_null(&n_hdlc->tx_buf_list.list,
+ struct n_hdlc_buf, list_item);
+ if (buf)
+ count += buf->count;
spin_unlock_irqrestore(&n_hdlc->tx_buf_list.spinlock,flags);
error = put_user(count, (int __user *)arg);
break;
poll_wait(filp, &tty->write_wait, wait);
/* set bits for operations that won't block */
- if (n_hdlc->rx_buf_list.head)
+ if (!list_empty(&n_hdlc->rx_buf_list.list))
mask |= POLLIN | POLLRDNORM; /* readable */
if (test_bit(TTY_OTHER_CLOSED, &tty->flags))
mask |= POLLHUP;
if (tty_hung_up_p(filp))
mask |= POLLHUP;
if (!tty_is_writelocked(tty) &&
- n_hdlc->tx_free_buf_list.head)
+ !list_empty(&n_hdlc->tx_free_buf_list.list))
mask |= POLLOUT | POLLWRNORM; /* writable */
}
return mask;
spin_lock_init(&n_hdlc->tx_free_buf_list.spinlock);
spin_lock_init(&n_hdlc->rx_buf_list.spinlock);
spin_lock_init(&n_hdlc->tx_buf_list.spinlock);
-
+
+ INIT_LIST_HEAD(&n_hdlc->rx_free_buf_list.list);
+ INIT_LIST_HEAD(&n_hdlc->tx_free_buf_list.list);
+ INIT_LIST_HEAD(&n_hdlc->rx_buf_list.list);
+ INIT_LIST_HEAD(&n_hdlc->tx_buf_list.list);
+
/* allocate free rx buffer list */
for(i=0;i<DEFAULT_RX_BUF_COUNT;i++) {
buf = kmalloc(N_HDLC_BUF_SIZE, GFP_KERNEL);
} /* end of n_hdlc_alloc() */
+/**
+ * n_hdlc_buf_return - put the HDLC buffer after the head of the specified list
+ * @buf_list - pointer to the buffer list
+ * @buf - pointer to the buffer
+ */
+static void n_hdlc_buf_return(struct n_hdlc_buf_list *buf_list,
+ struct n_hdlc_buf *buf)
+{
+ unsigned long flags;
+
+ spin_lock_irqsave(&buf_list->spinlock, flags);
+
+ list_add(&buf->list_item, &buf_list->list);
+ buf_list->count++;
+
+ spin_unlock_irqrestore(&buf_list->spinlock, flags);
+}
+
/**
* n_hdlc_buf_put - add specified HDLC buffer to tail of specified list
- * @list - pointer to buffer list
+ * @buf_list - pointer to buffer list
* @buf - pointer to buffer
*/
-static void n_hdlc_buf_put(struct n_hdlc_buf_list *list,
+static void n_hdlc_buf_put(struct n_hdlc_buf_list *buf_list,
struct n_hdlc_buf *buf)
{
unsigned long flags;
- spin_lock_irqsave(&list->spinlock,flags);
-
- buf->link=NULL;
- if (list->tail)
- list->tail->link = buf;
- else
- list->head = buf;
- list->tail = buf;
- (list->count)++;
-
- spin_unlock_irqrestore(&list->spinlock,flags);
-
+
+ spin_lock_irqsave(&buf_list->spinlock, flags);
+
+ list_add_tail(&buf->list_item, &buf_list->list);
+ buf_list->count++;
+
+ spin_unlock_irqrestore(&buf_list->spinlock, flags);
} /* end of n_hdlc_buf_put() */
/**
* n_hdlc_buf_get - remove and return an HDLC buffer from list
- * @list - pointer to HDLC buffer list
+ * @buf_list - pointer to HDLC buffer list
*
* Remove and return an HDLC buffer from the head of the specified HDLC buffer
* list.
* Returns a pointer to HDLC buffer if available, otherwise %NULL.
*/
-static struct n_hdlc_buf* n_hdlc_buf_get(struct n_hdlc_buf_list *list)
+static struct n_hdlc_buf *n_hdlc_buf_get(struct n_hdlc_buf_list *buf_list)
{
unsigned long flags;
struct n_hdlc_buf *buf;
- spin_lock_irqsave(&list->spinlock,flags);
-
- buf = list->head;
+
+ spin_lock_irqsave(&buf_list->spinlock, flags);
+
+ buf = list_first_entry_or_null(&buf_list->list,
+ struct n_hdlc_buf, list_item);
if (buf) {
- list->head = buf->link;
- (list->count)--;
+ list_del(&buf->list_item);
+ buf_list->count--;
}
- if (!list->head)
- list->tail = NULL;
-
- spin_unlock_irqrestore(&list->spinlock,flags);
+
+ spin_unlock_irqrestore(&buf_list->spinlock, flags);
return buf;
-
} /* end of n_hdlc_buf_get() */
static char hdlc_banner[] __initdata =
#include <linux/tty.h>
#include <linux/tty_flip.h>
#include <linux/fcntl.h>
-#include <linux/sched.h>
+#include <linux/sched/signal.h>
#include <linux/string.h>
#include <linux/major.h>
#include <linux/mm.h>
#include <linux/types.h>
#include <linux/errno.h>
#include <linux/signal.h>
-#include <linux/sched.h>
+#include <linux/sched/signal.h>
#include <linux/timer.h>
#include <linux/interrupt.h>
#include <linux/tty.h>
if (ourport->dma) {
ret = s3c24xx_serial_request_dma(ourport);
if (ret < 0) {
- dev_warn(port->dev, "DMA request failed\n");
- return ret;
+ dev_warn(port->dev,
+ "DMA request failed, DMA will not be used\n");
+ devm_kfree(port->dev, ourport->dma);
+ ourport->dma = NULL;
}
}
#include <linux/tty_flip.h>
#include <linux/spi/spi.h>
#include <linux/uaccess.h>
+#include <uapi/linux/sched/types.h>
#define SC16IS7XX_NAME "sc16is7xx"
#define SC16IS7XX_MAX_DEVS 8
#include <linux/tty.h>
#include <linux/tty_flip.h>
#include <linux/slab.h>
+#include <linux/sched/signal.h>
#include <linux/init.h>
#include <linux/console.h>
#include <linux/of.h>
#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
-#include <linux/sched.h>
+#include <linux/sched/signal.h>
#include <linux/sched/rt.h>
+#include <linux/sched/debug.h>
+#include <linux/sched/task.h>
#include <linux/interrupt.h>
#include <linux/mm.h>
#include <linux/fs.h>
#include <linux/errno.h>
#include <linux/signal.h>
#include <linux/fcntl.h>
-#include <linux/sched.h>
+#include <linux/sched/signal.h>
+#include <linux/sched/task.h>
#include <linux/interrupt.h>
#include <linux/tty.h>
#include <linux/tty_driver.h>
#include <linux/types.h>
#include <linux/termios.h>
#include <linux/errno.h>
-#include <linux/sched.h>
+#include <linux/sched/signal.h>
#include <linux/kernel.h>
#include <linux/major.h>
#include <linux/tty.h>
#include <linux/atomic.h>
#include <linux/tty.h>
#include <linux/sched.h>
+#include <linux/sched/debug.h>
+#include <linux/sched/task.h>
#ifdef CONFIG_DEBUG_LOCK_ALLOC
#include <linux/timer.h>
#include <linux/string.h>
#include <linux/slab.h>
-#include <linux/sched.h>
+#include <linux/sched/signal.h>
#include <linux/wait.h>
#include <linux/bitops.h>
#include <linux/delay.h>
#include <linux/consolemap.h>
#include <linux/module.h>
-#include <linux/sched.h>
+#include <linux/sched/signal.h>
+#include <linux/sched/debug.h>
+#include <linux/sched/debug.h>
#include <linux/tty.h>
#include <linux/tty_flip.h>
#include <linux/mm.h>
#include <linux/module.h>
#include <linux/types.h>
-#include <linux/sched.h>
+#include <linux/sched/signal.h>
#include <linux/tty.h>
#include <linux/tty_flip.h>
#include <linux/kernel.h>
#include <linux/types.h>
#include <linux/errno.h>
-#include <linux/sched.h>
+#include <linux/sched/signal.h>
#include <linux/tty.h>
#include <linux/timer.h>
#include <linux/kernel.h>
#include <linux/slab.h>
#include <linux/mm.h>
#include <linux/idr.h>
-#include <linux/sched.h>
+#include <linux/sched/signal.h>
#include <linux/string.h>
#include <linux/kobject.h>
#include <linux/cdev.h>
#include <linux/moduleparam.h>
#include <linux/netdevice.h>
#include <linux/proc_fs.h>
-#include <linux/sched.h>
+#include <linux/sched/signal.h>
#include <linux/signal.h>
#include <linux/slab.h>
#include <linux/stat.h>
#undef VERBOSE_DEBUG
#include <linux/kernel.h>
+#include <linux/sched/signal.h>
#include <linux/errno.h>
#include <linux/init.h>
#include <linux/slab.h>
#include <linux/module.h>
#include <linux/kernel.h>
-#include <linux/sched.h>
+#include <linux/sched/signal.h>
#include <linux/signal.h>
#include <linux/poll.h>
#include <linux/slab.h>
#include <linux/fs.h>
#include <linux/mm.h>
+#include <linux/sched/signal.h>
#include <linux/slab.h>
#include <linux/signal.h>
#include <linux/poll.h>
#include <linux/module.h>
#include <linux/moduleparam.h>
#include <linux/completion.h>
-#include <linux/sched.h>
+#include <linux/sched/mm.h>
#include <linux/list.h>
#include <linux/slab.h>
#include <linux/ioctl.h>
val = dwc3_omap_read_utmi_ctrl(omap);
val |= USBOTGSS_UTMI_OTG_CTRL_IDDIG;
dwc3_omap_write_utmi_ctrl(omap, val);
+ break;
case OMAP_DWC3_VBUS_OFF:
val = dwc3_omap_read_utmi_ctrl(omap);
{
u32 reg;
struct device_node *node = omap->dev->of_node;
- int utmi_mode = 0;
+ u32 utmi_mode = 0;
reg = dwc3_omap_read_utmi_ctrl(omap);
if (r == req) {
/* wait until it is processed */
dwc3_stop_active_transfer(dwc, dep->number, true);
+
+ /*
+ * If request was already started, this means we had to
+ * stop the transfer. With that we also need to ignore
+ * all TRBs used by the request, however TRBs can only
+ * be modified after completion of END_TRANSFER
+ * command. So what we do here is that we wait for
+ * END_TRANSFER completion and only after that, we jump
+ * over TRBs by clearing HWO and incrementing dequeue
+ * pointer.
+ *
+ * Note that we have 2 possible types of transfers here:
+ *
+ * i) Linear buffer request
+ * ii) SG-list based request
+ *
+ * SG-list based requests will have r->num_pending_sgs
+ * set to a valid number (> 0). Linear requests,
+ * normally use a single TRB.
+ *
+ * For each of these two cases, if r->unaligned flag is
+ * set, one extra TRB has been used to align transfer
+ * size to wMaxPacketSize.
+ *
+ * All of these cases need to be taken into
+ * consideration so we don't mess up our TRB ring
+ * pointers.
+ */
+ wait_event_lock_irq(dep->wait_end_transfer,
+ !(dep->flags & DWC3_EP_END_TRANSFER_PENDING),
+ dwc->lock);
+
+ if (!r->trb)
+ goto out1;
+
+ if (r->num_pending_sgs) {
+ struct dwc3_trb *trb;
+ int i = 0;
+
+ for (i = 0; i < r->num_pending_sgs; i++) {
+ trb = r->trb + i;
+ trb->ctrl &= ~DWC3_TRB_CTRL_HWO;
+ dwc3_ep_inc_deq(dep);
+ }
+
+ if (r->unaligned) {
+ trb = r->trb + r->num_pending_sgs + 1;
+ trb->ctrl &= ~DWC3_TRB_CTRL_HWO;
+ dwc3_ep_inc_deq(dep);
+ }
+ } else {
+ struct dwc3_trb *trb = r->trb;
+
+ trb->ctrl &= ~DWC3_TRB_CTRL_HWO;
+ dwc3_ep_inc_deq(dep);
+
+ if (r->unaligned) {
+ trb = r->trb + 1;
+ trb->ctrl &= ~DWC3_TRB_CTRL_HWO;
+ dwc3_ep_inc_deq(dep);
+ }
+ }
goto out1;
}
dev_err(dwc->dev, "request %p was not queued to %s\n",
out1:
/* giveback the request */
+ dep->queued_requests--;
dwc3_gadget_giveback(dep, req, -ECONNRESET);
out0:
return 1;
}
- if ((trb->ctrl & DWC3_TRB_CTRL_HWO) && status != -ESHUTDOWN)
- return 1;
-
count = trb->size & DWC3_TRB_SIZE_MASK;
req->remaining += count;
+ if ((trb->ctrl & DWC3_TRB_CTRL_HWO) && status != -ESHUTDOWN)
+ return 1;
+
if (dep->direction) {
if (count) {
trb_status = DWC3_TRB_SIZE_TRBSTS(trb->size);
int dwc3_gadget_suspend(struct dwc3 *dwc)
{
- int ret;
-
if (!dwc->gadget_driver)
return 0;
- ret = dwc3_gadget_run_stop(dwc, false, false);
- if (ret < 0)
- return ret;
-
+ dwc3_gadget_run_stop(dwc, false, false);
dwc3_disconnect_gadget(dwc);
__dwc3_gadget_stop(dwc);
#define gadget_to_dwc(g) (container_of(g, struct dwc3, gadget))
/* DEPCFG parameter 1 */
-#define DWC3_DEPCFG_INT_NUM(n) ((n) << 0)
+#define DWC3_DEPCFG_INT_NUM(n) (((n) & 0x1f) << 0)
#define DWC3_DEPCFG_XFER_COMPLETE_EN (1 << 8)
#define DWC3_DEPCFG_XFER_IN_PROGRESS_EN (1 << 9)
#define DWC3_DEPCFG_XFER_NOT_READY_EN (1 << 10)
#define DWC3_DEPCFG_FIFO_ERROR_EN (1 << 11)
#define DWC3_DEPCFG_STREAM_EVENT_EN (1 << 13)
-#define DWC3_DEPCFG_BINTERVAL_M1(n) ((n) << 16)
+#define DWC3_DEPCFG_BINTERVAL_M1(n) (((n) & 0xff) << 16)
#define DWC3_DEPCFG_STREAM_CAPABLE (1 << 24)
-#define DWC3_DEPCFG_EP_NUMBER(n) ((n) << 25)
+#define DWC3_DEPCFG_EP_NUMBER(n) (((n) & 0x1f) << 25)
#define DWC3_DEPCFG_BULK_BASED (1 << 30)
#define DWC3_DEPCFG_FIFO_BASED (1 << 31)
/* DEPCFG parameter 0 */
-#define DWC3_DEPCFG_EP_TYPE(n) ((n) << 1)
-#define DWC3_DEPCFG_MAX_PACKET_SIZE(n) ((n) << 3)
-#define DWC3_DEPCFG_FIFO_NUMBER(n) ((n) << 17)
-#define DWC3_DEPCFG_BURST_SIZE(n) ((n) << 22)
+#define DWC3_DEPCFG_EP_TYPE(n) (((n) & 0x3) << 1)
+#define DWC3_DEPCFG_MAX_PACKET_SIZE(n) (((n) & 0x7ff) << 3)
+#define DWC3_DEPCFG_FIFO_NUMBER(n) (((n) & 0x1f) << 17)
+#define DWC3_DEPCFG_BURST_SIZE(n) (((n) & 0xf) << 22)
#define DWC3_DEPCFG_DATA_SEQ_NUM(n) ((n) << 26)
/* This applies for core versions earlier than 1.94a */
#define DWC3_DEPCFG_IGN_SEQ_NUM (1 << 31)
ret = unregister_gadget(gi);
if (ret)
goto err;
+ kfree(name);
} else {
if (gi->composite.gadget_driver.udc_name) {
ret = -EBUSY;
#include <linux/export.h>
#include <linux/hid.h>
#include <linux/module.h>
+#include <linux/sched/signal.h>
#include <linux/uio.h>
#include <asm/unaligned.h>
spin_lock_irqsave(&func->ffs->eps_lock, flags);
while(count--) {
struct usb_endpoint_descriptor *ds;
+ struct usb_ss_ep_comp_descriptor *comp_desc = NULL;
+ int needs_comp_desc = false;
int desc_idx;
- if (ffs->gadget->speed == USB_SPEED_SUPER)
+ if (ffs->gadget->speed == USB_SPEED_SUPER) {
desc_idx = 2;
- else if (ffs->gadget->speed == USB_SPEED_HIGH)
+ needs_comp_desc = true;
+ } else if (ffs->gadget->speed == USB_SPEED_HIGH)
desc_idx = 1;
else
desc_idx = 0;
ep->ep->driver_data = ep;
ep->ep->desc = ds;
+
+ comp_desc = (struct usb_ss_ep_comp_descriptor *)(ds +
+ USB_DT_ENDPOINT_SIZE);
+ ep->ep->maxburst = comp_desc->bMaxBurst + 1;
+
+ if (needs_comp_desc)
+ ep->ep->comp_desc = comp_desc;
+
ret = usb_ep_enable(ep->ep);
if (likely(!ret)) {
epfile->ep = ep;
if (len < sizeof(*d) ||
d->bFirstInterfaceNumber >= ffs->interfaces_count ||
- d->Reserved1)
+ !d->Reserved1)
return -EINVAL;
for (i = 0; i < ARRAY_SIZE(d->Reserved2); ++i)
if (d->Reserved2[i])
#include <linux/fs.h>
#include <linux/kref.h>
#include <linux/kthread.h>
+#include <linux/sched/signal.h>
#include <linux/limits.h>
#include <linux/rwsem.h>
#include <linux/slab.h>
memcpy(&uvc_event->req, ctrl, sizeof(uvc_event->req));
v4l2_event_queue(&uvc->vdev, &v4l2_event);
- /* Pass additional setup data to userspace */
- if (uvc->event_setup_out && uvc->event_length) {
- uvc->control_req->length = uvc->event_length;
- return usb_ep_queue(uvc->func.config->cdev->gadget->ep0,
- uvc->control_req, GFP_ATOMIC);
- }
-
return 0;
}
/* /dev/gadget/$CHIP represents ep0 and the whole device */
enum ep0_state {
- /* DISBLED is the initial state.
- */
+ /* DISABLED is the initial state. */
STATE_DEV_DISABLED = 0,
/* Only one open() of /dev/gadget/$CHIP; only one file tracks
spin_lock_irq (&dev->lock);
value = -EINVAL;
- if (dev->buf)
+ if (dev->buf) {
+ kfree(kbuf);
goto fail;
+ }
dev->buf = kbuf;
/* full or low speed config */
{
struct usba_ep *ep = to_usba_ep(_ep);
struct usba_udc *udc = ep->udc;
- unsigned long flags, ept_cfg, maxpacket;
+ unsigned long flags, maxpacket;
unsigned int nr_trans;
DBG(DBG_GADGET, "%s: ep_enable: desc=%p\n", ep->ep.name, desc);
ep->is_in = 0;
DBG(DBG_ERR, "%s: EPT_CFG = 0x%lx (maxpacket = %lu)\n",
- ep->ep.name, ept_cfg, maxpacket);
+ ep->ep.name, ep->ept_cfg, maxpacket);
if (usb_endpoint_dir_in(desc)) {
ep->is_in = 1;
int rc;
dum = *((void **)dev_get_platdata(&pdev->dev));
+ /* Clear usb_gadget region for new registration to udc-core */
+ memzero_explicit(&dum->gadget, sizeof(struct usb_gadget));
dum->gadget.name = gadget_name;
dum->gadget.ops = &dummy_ops;
dum->gadget.max_speed = USB_SPEED_SUPER;
*/
while (!list_empty(&ep->queue)) {
struct net2280_request *req;
- u32 tmp;
+ u32 req_dma_count;
req = list_entry(ep->queue.next,
struct net2280_request, queue);
if (!req->valid)
break;
rmb();
- tmp = le32_to_cpup(&req->td->dmacount);
- if ((tmp & BIT(VALID_BIT)) != 0)
+ req_dma_count = le32_to_cpup(&req->td->dmacount);
+ if ((req_dma_count & BIT(VALID_BIT)) != 0)
break;
/* SHORT_PACKET_TRANSFERRED_INTERRUPT handles "usb-short"
*/
if (unlikely(req->td->dmadesc == 0)) {
/* paranoia */
- tmp = readl(&ep->dma->dmacount);
- if (tmp & DMA_BYTE_COUNT_MASK)
+ u32 const ep_dmacount = readl(&ep->dma->dmacount);
+
+ if (ep_dmacount & DMA_BYTE_COUNT_MASK)
break;
/* single transfer mode */
- dma_done(ep, req, tmp, 0);
+ dma_done(ep, req, req_dma_count, 0);
num_completed++;
break;
} else if (!ep->is_in &&
(req->req.length % ep->ep.maxpacket) &&
!(ep->dev->quirks & PLX_PCIE)) {
- tmp = readl(&ep->regs->ep_stat);
+ u32 const ep_stat = readl(&ep->regs->ep_stat);
/* AVOID TROUBLE HERE by not issuing short reads from
* your gadget driver. That helps avoids errata 0121,
* 0122, and 0124; not all cases trigger the warning.
*/
- if ((tmp & BIT(NAK_OUT_PACKETS)) == 0) {
+ if ((ep_stat & BIT(NAK_OUT_PACKETS)) == 0) {
ep_warn(ep->dev, "%s lost packet sync!\n",
ep->ep.name);
req->req.status = -EOVERFLOW;
} else {
- tmp = readl(&ep->regs->ep_avail);
- if (tmp) {
+ u32 const ep_avail = readl(&ep->regs->ep_avail);
+ if (ep_avail) {
/* fifo gets flushed later */
ep->out_overflow = 1;
ep_dbg(ep->dev,
"%s dma, discard %d len %d\n",
- ep->ep.name, tmp,
+ ep->ep.name, ep_avail,
req->req.length);
req->req.status = -EOVERFLOW;
}
}
}
- dma_done(ep, req, tmp, 0);
+ dma_done(ep, req, req_dma_count, 0);
num_completed++;
}
usb_del_gadget_udc(&udc->gadget);
pxa_cleanup_debugfs(udc);
- if (!IS_ERR_OR_NULL(udc->transceiver))
+ if (!IS_ERR_OR_NULL(udc->transceiver)) {
usb_unregister_notifier(udc->transceiver, &pxa27x_udc_phy);
- usb_put_phy(udc->transceiver);
+ usb_put_phy(udc->transceiver);
+ }
udc->transceiver = NULL;
the_controller = NULL;
case USB_PORT_FEAT_SUSPEND:
dev_dbg(hcd->self.controller, "SetPortFeat: SUSPEND\n");
- if (valid_port(wIndex)) {
+ if (valid_port(wIndex) && ohci_at91->sfr_regmap) {
ohci_at91_port_suspend(ohci_at91->sfr_regmap,
1);
return 0;
case USB_PORT_FEAT_SUSPEND:
dev_dbg(hcd->self.controller, "ClearPortFeature: SUSPEND\n");
- if (valid_port(wIndex)) {
+ if (valid_port(wIndex) && ohci_at91->sfr_regmap) {
ohci_at91_port_suspend(ohci_at91->sfr_regmap,
0);
return 0;
xhci_dbg(xhci, "RTSOFF 0x%x:\n", temp & RTSOFF_MASK);
/* xhci 1.1 controllers have the HCCPARAMS2 register */
- if (hci_version > 100) {
+ if (hci_version > 0x100) {
temp = readl(&xhci->cap_regs->hcc_params2);
xhci_dbg(xhci, "HCC PARAMS2 0x%x:\n", (unsigned int) temp);
xhci_dbg(xhci, " HC %s Force save context capability",
static int xhci_mtk_setup(struct usb_hcd *hcd);
static const struct xhci_driver_overrides xhci_mtk_overrides __initconst = {
- .extra_priv_size = sizeof(struct xhci_hcd),
.reset = xhci_mtk_setup,
};
goto power_off_phys;
}
- if (HCC_MAX_PSA(xhci->hcc_params) >= 4)
- xhci->shared_hcd->can_do_streams = 1;
-
ret = usb_add_hcd(hcd, irq, IRQF_SHARED);
if (ret)
goto put_usb3_hcd;
+ if (HCC_MAX_PSA(xhci->hcc_params) >= 4)
+ xhci->shared_hcd->can_do_streams = 1;
+
ret = usb_add_hcd(xhci->shared_hcd, irq, IRQF_SHARED);
if (ret)
goto dealloc_usb2_hcd;
struct xhci_hcd *xhci = hcd_to_xhci(hcd);
struct clk *clk = xhci->clk;
+ xhci->xhc_state |= XHCI_STATE_REMOVING;
+
usb_remove_hcd(xhci->shared_hcd);
usb_phy_shutdown(hcd->usb_phy);
}
static const struct xhci_driver_overrides tegra_xhci_overrides __initconst = {
- .extra_priv_size = sizeof(struct xhci_hcd),
.reset = tegra_xhci_setup,
};
spin_lock_irqsave(&xhci->lock, flags);
- /* disble usb3 ports Wake bits*/
+ /* disable usb3 ports Wake bits */
port_index = xhci->num_usb3_ports;
port_array = xhci->usb3_ports;
while (port_index--) {
writel(t2, port_array[port_index]);
}
- /* disble usb2 ports Wake bits*/
+ /* disable usb2 ports Wake bits */
port_index = xhci->num_usb2_ports;
port_array = xhci->usb2_ports;
while (port_index--) {
* (20/10/1999)
*/
-#include <linux/sched.h>
+#include <linux/sched/signal.h>
#include <linux/signal.h>
#include <linux/spinlock.h>
#include <linux/errno.h>
#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
#include <linux/kernel.h>
+#include <linux/sched/signal.h>
#include <linux/errno.h>
#include <linux/slab.h>
#include <linux/module.h>
*/
#include <linux/kernel.h>
+#include <linux/sched/signal.h>
#include <linux/errno.h>
#include <linux/delay.h>
#include <linux/slab.h>
iface_desc = interface->cur_altsetting;
dev->product_id = le16_to_cpu(udev->descriptor.idProduct);
- if (iface_desc->desc.bNumEndpoints < 1) {
- dev_err(&interface->dev, "Invalid number of endpoints\n");
- retval = -EINVAL;
- goto error;
- }
-
/* set up the endpoint information */
for (i = 0; i < iface_desc->desc.bNumEndpoints; ++i) {
endpoint = &iface_desc->endpoint[i].desc;
/* this one will match for the IOWarrior56 only */
dev->int_out_endpoint = endpoint;
}
+
+ if (!dev->int_in_endpoint) {
+ dev_err(&interface->dev, "no interrupt-in endpoint found\n");
+ retval = -ENODEV;
+ goto error;
+ }
+
+ if (dev->product_id == USB_DEVICE_ID_CODEMERCS_IOW56) {
+ if (!dev->int_out_endpoint) {
+ dev_err(&interface->dev, "no interrupt-out endpoint found\n");
+ retval = -ENODEV;
+ goto error;
+ }
+ }
+
/* we have to check the report_size often, so remember it in the endianness suitable for our machine */
dev->report_size = usb_endpoint_maxp(dev->int_in_endpoint);
if ((dev->interface->cur_altsetting->desc.bInterfaceNumber == 0) &&
#include <linux/module.h>
#include <linux/kernel.h>
#include <linux/signal.h>
-#include <linux/sched.h>
+#include <linux/sched/signal.h>
#include <linux/mutex.h>
#include <linux/errno.h>
#include <linux/random.h>
if (of_get_property(np, "dynamic-power-switching", NULL))
hub->conf_data2 |= BIT(7);
- if (of_get_property(np, "oc-delay-100us", NULL)) {
- hub->conf_data2 &= ~BIT(5);
- hub->conf_data2 &= ~BIT(4);
- } else if (of_get_property(np, "oc-delay-4ms", NULL)) {
- hub->conf_data2 &= ~BIT(5);
- hub->conf_data2 |= BIT(4);
- } else if (of_get_property(np, "oc-delay-8ms", NULL)) {
- hub->conf_data2 |= BIT(5);
- hub->conf_data2 &= ~BIT(4);
- } else if (of_get_property(np, "oc-delay-16ms", NULL)) {
- hub->conf_data2 |= BIT(5);
- hub->conf_data2 |= BIT(4);
+ if (!of_property_read_u32(np, "oc-delay-us", property_u32)) {
+ if (*property_u32 == 100) {
+ /* 100 us*/
+ hub->conf_data2 &= ~BIT(5);
+ hub->conf_data2 &= ~BIT(4);
+ } else if (*property_u32 == 4000) {
+ /* 4 ms */
+ hub->conf_data2 &= ~BIT(5);
+ hub->conf_data2 |= BIT(4);
+ } else if (*property_u32 == 16000) {
+ /* 16 ms */
+ hub->conf_data2 |= BIT(5);
+ hub->conf_data2 |= BIT(4);
+ } else {
+ /* 8 ms (DEFAULT) */
+ hub->conf_data2 |= BIT(5);
+ hub->conf_data2 &= ~BIT(4);
+ }
}
if (of_get_property(np, "compound-device", NULL))
}
}
- hub->max_power_sp = USB251XB_DEF_MAX_POWER_SELF;
- if (!of_property_read_u32(np, "max-sp-power", property_u32))
- hub->max_power_sp = min_t(u8, be32_to_cpu(*property_u32) / 2,
- 250);
-
- hub->max_power_bp = USB251XB_DEF_MAX_POWER_BUS;
- if (!of_property_read_u32(np, "max-bp-power", property_u32))
- hub->max_power_bp = min_t(u8, be32_to_cpu(*property_u32) / 2,
- 250);
-
- hub->max_current_sp = USB251XB_DEF_MAX_CURRENT_SELF;
- if (!of_property_read_u32(np, "max-sp-current", property_u32))
- hub->max_current_sp = min_t(u8, be32_to_cpu(*property_u32) / 2,
- 250);
-
- hub->max_current_bp = USB251XB_DEF_MAX_CURRENT_BUS;
- if (!of_property_read_u32(np, "max-bp-current", property_u32))
- hub->max_current_bp = min_t(u8, be32_to_cpu(*property_u32) / 2,
- 250);
-
hub->power_on_time = USB251XB_DEF_POWER_ON_TIME;
- if (!of_property_read_u32(np, "power-on-time", property_u32))
- hub->power_on_time = min_t(u8, be32_to_cpu(*property_u32) / 2,
- 255);
+ if (!of_property_read_u32(np, "power-on-time-ms", property_u32))
+ hub->power_on_time = min_t(u8, *property_u32 / 2, 255);
if (of_property_read_u16_array(np, "language-id", &hub->lang_id, 1))
hub->lang_id = USB251XB_DEF_LANGUAGE_ID;
/* The following parameters are currently not exposed to devicetree, but
* may be as soon as needed.
*/
+ hub->max_power_sp = USB251XB_DEF_MAX_POWER_SELF;
+ hub->max_power_bp = USB251XB_DEF_MAX_POWER_BUS;
+ hub->max_current_sp = USB251XB_DEF_MAX_CURRENT_SELF;
+ hub->max_current_bp = USB251XB_DEF_MAX_CURRENT_BUS;
hub->bat_charge_en = USB251XB_DEF_BATTERY_CHARGING_ENABLE;
hub->boost_up = USB251XB_DEF_BOOST_UP;
hub->boost_x = USB251XB_DEF_BOOST_X;
#include <linux/completion.h>
#include <linux/kref.h>
#include <linux/slab.h>
+#include <linux/sched/signal.h>
/*
* Version Information
*/
#include <linux/kernel.h>
+#include <linux/sched/signal.h>
#include <linux/types.h>
#include <linux/fs.h>
#include <linux/cdev.h>
#include <linux/list.h>
#include <linux/usb.h>
#include <linux/slab.h>
+#include <linux/sched/signal.h>
#include <linux/time.h>
#include <linux/ktime.h>
#include <linux/export.h>
};
MODULE_DEVICE_TABLE(i2c, isp1301_id);
+static const struct of_device_id isp1301_of_match[] = {
+ {.compatible = "nxp,isp1301" },
+ { },
+};
+MODULE_DEVICE_TABLE(of, isp1301_of_match);
+
static struct i2c_client *isp1301_i2c_client;
static int __isp1301_write(struct isp1301 *isp, u8 reg, u8 value, u8 clear)
static struct i2c_driver isp1301_driver = {
.driver = {
.name = DRV_NAME,
+ .of_match_table = of_match_ptr(isp1301_of_match),
},
.probe = isp1301_probe,
.remove = isp1301_remove,
#include <linux/uaccess.h>
#include <linux/usb.h>
#include <linux/wait.h>
+#include <linux/sched/signal.h>
#include <linux/usb/serial.h>
/* Defines */
return -1;
/* handle each oob command */
- for (i = 0; i < urb->actual_length - 4; i += 4) {
+ for (i = 0; i < urb->actual_length - 3; i += 4) {
opcode = buf[i];
line = buf[i + 1];
status = buf[i + 2];
*/
#include <linux/kernel.h>
+#include <linux/sched/signal.h>
#include <linux/errno.h>
#include <linux/slab.h>
#include <linux/sysrq.h>
function = TIUMP_GET_FUNC_FROM_CODE(data[0]);
dev_dbg(dev, "%s - port_number %d, function %d, info 0x%x\n", __func__,
port_number, function, data[1]);
+
+ if (port_number >= edge_serial->serial->num_ports) {
+ dev_err(dev, "bad port number %d\n", port_number);
+ goto exit;
+ }
+
port = edge_serial->serial->port[port_number];
edge_port = usb_get_serial_port_data(port);
if (!edge_port) {
port_number = edge_port->port->port_number;
- if (edge_port->lsr_event) {
+ if (urb->actual_length > 0 && edge_port->lsr_event) {
edge_port->lsr_event = 0;
dev_dbg(dev, "%s ===== Port %u LSR Status = %02x, Data = %02x ======\n",
__func__, port_number, edge_port->lsr_mask, *data);
#define BT_IGNITIONPRO_ID 0x2000
/* function prototypes */
-static int omninet_open(struct tty_struct *tty, struct usb_serial_port *port);
static void omninet_process_read_urb(struct urb *urb);
static void omninet_write_bulk_callback(struct urb *urb);
static int omninet_write(struct tty_struct *tty, struct usb_serial_port *port,
.attach = omninet_attach,
.port_probe = omninet_port_probe,
.port_remove = omninet_port_remove,
- .open = omninet_open,
.write = omninet_write,
.write_room = omninet_write_room,
.write_bulk_callback = omninet_write_bulk_callback,
return 0;
}
-static int omninet_open(struct tty_struct *tty, struct usb_serial_port *port)
-{
- struct usb_serial *serial = port->serial;
- struct usb_serial_port *wport;
-
- wport = serial->port[1];
- tty_port_tty_set(&wport->port, tty);
-
- return usb_serial_generic_open(tty, port);
-}
-
#define OMNINET_HEADERLEN 4
#define OMNINET_BULKOUTSIZE 64
#define OMNINET_PAYLOADSIZE (OMNINET_BULKOUTSIZE - OMNINET_HEADERLEN)
if (!safe)
goto out;
+ if (length < 2) {
+ dev_err(&port->dev, "malformed packet\n");
+ return;
+ }
+
fcs = fcs_compute10(data, length, CRC10_INITFCS);
if (fcs) {
dev_err(&port->dev, "%s - bad CRC %x\n", __func__, fcs);
USB_SC_DEVICE, USB_PR_DEVICE, NULL,
US_FL_IGNORE_RESIDUE ),
+/*
+ * Reported by Tobias Jakobi <tjakobi@math.uni-bielefeld.de>
+ * The INIC-3619 bridge is used in the StarTech SLSODDU33B
+ * SATA-USB enclosure for slimline optical drives.
+ *
+ * The quirk enables MakeMKV to properly exchange keys with
+ * an installed BD drive.
+ */
+UNUSUAL_DEV( 0x13fd, 0x3609, 0x0209, 0x0209,
+ "Initio Corporation",
+ "INIC-3619",
+ USB_SC_DEVICE, USB_PR_DEVICE, NULL,
+ US_FL_IGNORE_RESIDUE ),
+
/* Reported by Qinglin Ye <yestyle@gmail.com> */
UNUSUAL_DEV( 0x13fe, 0x3600, 0x0100, 0x0100,
"Kingston",
int usbip_recv(struct socket *sock, void *buf, int size)
{
int result;
- struct msghdr msg;
- struct kvec iov;
+ struct kvec iov = {.iov_base = buf, .iov_len = size};
+ struct msghdr msg = {.msg_flags = MSG_NOSIGNAL};
int total = 0;
- /* for blocks of if (usbip_dbg_flag_xmit) */
- char *bp = buf;
- int osize = size;
+ iov_iter_kvec(&msg.msg_iter, READ|ITER_KVEC, &iov, 1, size);
usbip_dbg_xmit("enter\n");
}
do {
+ int sz = msg_data_left(&msg);
sock->sk->sk_allocation = GFP_NOIO;
- iov.iov_base = buf;
- iov.iov_len = size;
- msg.msg_name = NULL;
- msg.msg_namelen = 0;
- msg.msg_control = NULL;
- msg.msg_controllen = 0;
- msg.msg_flags = MSG_NOSIGNAL;
-
- result = kernel_recvmsg(sock, &msg, &iov, 1, size, MSG_WAITALL);
+
+ result = sock_recvmsg(sock, &msg, MSG_WAITALL);
if (result <= 0) {
pr_debug("receive sock %p buf %p size %u ret %d total %d\n",
- sock, buf, size, result, total);
+ sock, buf + total, sz, result, total);
goto err;
}
- size -= result;
- buf += result;
total += result;
- } while (size > 0);
+ } while (msg_data_left(&msg));
if (usbip_dbg_flag_xmit) {
if (!in_interrupt())
pr_debug("interrupt :");
pr_debug("receiving....\n");
- usbip_dump_buffer(bp, osize);
- pr_debug("received, osize %d ret %d size %d total %d\n",
- osize, result, size, total);
+ usbip_dump_buffer(buf, size);
+ pr_debug("received, osize %d ret %d size %zd total %d\n",
+ size, result, msg_data_left(&msg), total);
}
return total;
#include <linux/types.h>
#include <linux/usb.h>
#include <linux/wait.h>
+#include <linux/sched/task.h>
#include <uapi/linux/usbip.h>
#define USBIP_VERSION "1.0.0"
#include <linux/err.h>
#include <linux/vfio.h>
#include <linux/vmalloc.h>
+#include <linux/sched/mm.h>
+#include <linux/sched/signal.h>
+
#include <asm/iommu.h>
#include <asm/tce.h>
#include <asm/mmu_context.h>
#include <linux/module.h>
#include <linux/mm.h>
#include <linux/rbtree.h>
-#include <linux/sched.h>
+#include <linux/sched/signal.h>
+#include <linux/sched/mm.h>
#include <linux/slab.h>
#include <linux/uaccess.h>
#include <linux/vfio.h>
#include <linux/workqueue.h>
#include <linux/file.h>
#include <linux/slab.h>
+#include <linux/sched/clock.h>
+#include <linux/sched/signal.h>
#include <linux/vmalloc.h>
#include <linux/net.h>
#include <linux/cgroup.h>
#include <linux/module.h>
#include <linux/sort.h>
+#include <linux/sched/mm.h>
+#include <linux/sched/signal.h>
#include <linux/interval_tree_generic.h>
#include "vhost.h"
}
EXPORT_SYMBOL_GPL(vhost_poll_queue);
+static void __vhost_vq_meta_reset(struct vhost_virtqueue *vq)
+{
+ int j;
+
+ for (j = 0; j < VHOST_NUM_ADDRS; j++)
+ vq->meta_iotlb[j] = NULL;
+}
+
+static void vhost_vq_meta_reset(struct vhost_dev *d)
+{
+ int i;
+
+ for (i = 0; i < d->nvqs; ++i)
+ __vhost_vq_meta_reset(d->vqs[i]);
+}
+
static void vhost_vq_reset(struct vhost_dev *dev,
struct vhost_virtqueue *vq)
{
vq->busyloop_timeout = 0;
vq->umem = NULL;
vq->iotlb = NULL;
+ __vhost_vq_meta_reset(vq);
}
static int vhost_worker(void *data)
return 1;
}
+static inline void __user *vhost_vq_meta_fetch(struct vhost_virtqueue *vq,
+ u64 addr, unsigned int size,
+ int type)
+{
+ const struct vhost_umem_node *node = vq->meta_iotlb[type];
+
+ if (!node)
+ return NULL;
+
+ return (void *)(uintptr_t)(node->userspace_addr + addr - node->start);
+}
+
/* Can we switch to this memory table? */
/* Caller should have device mutex but not vq mutex */
static int memory_access_ok(struct vhost_dev *d, struct vhost_umem *umem,
* could be access through iotlb. So -EAGAIN should
* not happen in this case.
*/
- /* TODO: more fast path */
struct iov_iter t;
+ void __user *uaddr = vhost_vq_meta_fetch(vq,
+ (u64)(uintptr_t)to, size,
+ VHOST_ADDR_DESC);
+
+ if (uaddr)
+ return __copy_to_user(uaddr, from, size);
+
ret = translate_desc(vq, (u64)(uintptr_t)to, size, vq->iotlb_iov,
ARRAY_SIZE(vq->iotlb_iov),
VHOST_ACCESS_WO);
* could be access through iotlb. So -EAGAIN should
* not happen in this case.
*/
- /* TODO: more fast path */
+ void __user *uaddr = vhost_vq_meta_fetch(vq,
+ (u64)(uintptr_t)from, size,
+ VHOST_ADDR_DESC);
struct iov_iter f;
+
+ if (uaddr)
+ return __copy_from_user(to, uaddr, size);
+
ret = translate_desc(vq, (u64)(uintptr_t)from, size, vq->iotlb_iov,
ARRAY_SIZE(vq->iotlb_iov),
VHOST_ACCESS_RO);
return ret;
}
-static void __user *__vhost_get_user(struct vhost_virtqueue *vq,
- void __user *addr, unsigned size)
+static void __user *__vhost_get_user_slow(struct vhost_virtqueue *vq,
+ void __user *addr, unsigned int size,
+ int type)
{
int ret;
- /* This function should be called after iotlb
- * prefetch, which means we're sure that vq
- * could be access through iotlb. So -EAGAIN should
- * not happen in this case.
- */
- /* TODO: more fast path */
ret = translate_desc(vq, (u64)(uintptr_t)addr, size, vq->iotlb_iov,
ARRAY_SIZE(vq->iotlb_iov),
VHOST_ACCESS_RO);
return vq->iotlb_iov[0].iov_base;
}
-#define vhost_put_user(vq, x, ptr) \
+/* This function should be called after iotlb
+ * prefetch, which means we're sure that vq
+ * could be access through iotlb. So -EAGAIN should
+ * not happen in this case.
+ */
+static inline void __user *__vhost_get_user(struct vhost_virtqueue *vq,
+ void *addr, unsigned int size,
+ int type)
+{
+ void __user *uaddr = vhost_vq_meta_fetch(vq,
+ (u64)(uintptr_t)addr, size, type);
+ if (uaddr)
+ return uaddr;
+
+ return __vhost_get_user_slow(vq, addr, size, type);
+}
+
+#define vhost_put_user(vq, x, ptr) \
({ \
int ret = -EFAULT; \
if (!vq->iotlb) { \
ret = __put_user(x, ptr); \
} else { \
__typeof__(ptr) to = \
- (__typeof__(ptr)) __vhost_get_user(vq, ptr, sizeof(*ptr)); \
+ (__typeof__(ptr)) __vhost_get_user(vq, ptr, \
+ sizeof(*ptr), VHOST_ADDR_USED); \
if (to != NULL) \
ret = __put_user(x, to); \
else \
ret; \
})
-#define vhost_get_user(vq, x, ptr) \
+#define vhost_get_user(vq, x, ptr, type) \
({ \
int ret; \
if (!vq->iotlb) { \
ret = __get_user(x, ptr); \
} else { \
__typeof__(ptr) from = \
- (__typeof__(ptr)) __vhost_get_user(vq, ptr, sizeof(*ptr)); \
+ (__typeof__(ptr)) __vhost_get_user(vq, ptr, \
+ sizeof(*ptr), \
+ type); \
if (from != NULL) \
ret = __get_user(x, from); \
else \
ret; \
})
+#define vhost_get_avail(vq, x, ptr) \
+ vhost_get_user(vq, x, ptr, VHOST_ADDR_AVAIL)
+
+#define vhost_get_used(vq, x, ptr) \
+ vhost_get_user(vq, x, ptr, VHOST_ADDR_USED)
+
static void vhost_dev_lock_vqs(struct vhost_dev *d)
{
int i = 0;
ret = -EFAULT;
break;
}
+ vhost_vq_meta_reset(dev);
if (vhost_new_umem_range(dev->iotlb, msg->iova, msg->size,
msg->iova + msg->size - 1,
msg->uaddr, msg->perm)) {
vhost_iotlb_notify_vq(dev, msg);
break;
case VHOST_IOTLB_INVALIDATE:
+ vhost_vq_meta_reset(dev);
vhost_del_umem_range(dev->iotlb, msg->iova,
msg->iova + msg->size - 1);
break;
sizeof *used + num * sizeof *used->ring + s);
}
+static void vhost_vq_meta_update(struct vhost_virtqueue *vq,
+ const struct vhost_umem_node *node,
+ int type)
+{
+ int access = (type == VHOST_ADDR_USED) ?
+ VHOST_ACCESS_WO : VHOST_ACCESS_RO;
+
+ if (likely(node->perm & access))
+ vq->meta_iotlb[type] = node;
+}
+
static int iotlb_access_ok(struct vhost_virtqueue *vq,
- int access, u64 addr, u64 len)
+ int access, u64 addr, u64 len, int type)
{
const struct vhost_umem_node *node;
struct vhost_umem *umem = vq->iotlb;
- u64 s = 0, size;
+ u64 s = 0, size, orig_addr = addr;
+
+ if (vhost_vq_meta_fetch(vq, addr, len, type))
+ return true;
while (len > s) {
node = vhost_umem_interval_tree_iter_first(&umem->umem_tree,
}
size = node->size - addr + node->start;
+
+ if (orig_addr == addr && size >= len)
+ vhost_vq_meta_update(vq, node, type);
+
s += size;
addr += size;
}
return 1;
return iotlb_access_ok(vq, VHOST_ACCESS_RO, (u64)(uintptr_t)vq->desc,
- num * sizeof *vq->desc) &&
+ num * sizeof(*vq->desc), VHOST_ADDR_DESC) &&
iotlb_access_ok(vq, VHOST_ACCESS_RO, (u64)(uintptr_t)vq->avail,
sizeof *vq->avail +
- num * sizeof *vq->avail->ring + s) &&
+ num * sizeof(*vq->avail->ring) + s,
+ VHOST_ADDR_AVAIL) &&
iotlb_access_ok(vq, VHOST_ACCESS_WO, (u64)(uintptr_t)vq->used,
sizeof *vq->used +
- num * sizeof *vq->used->ring + s);
+ num * sizeof(*vq->used->ring) + s,
+ VHOST_ADDR_USED);
}
EXPORT_SYMBOL_GPL(vq_iotlb_prefetch);
r = -EFAULT;
goto err;
}
- r = vhost_get_user(vq, last_used_idx, &vq->used->idx);
+ r = vhost_get_used(vq, last_used_idx, &vq->used->idx);
if (r) {
vq_err(vq, "Can't access used idx at %p\n",
&vq->used->idx);
/* Check it isn't doing very strange things with descriptor numbers. */
last_avail_idx = vq->last_avail_idx;
- if (unlikely(vhost_get_user(vq, avail_idx, &vq->avail->idx))) {
- vq_err(vq, "Failed to access avail idx at %p\n",
- &vq->avail->idx);
- return -EFAULT;
- }
- vq->avail_idx = vhost16_to_cpu(vq, avail_idx);
- if (unlikely((u16)(vq->avail_idx - last_avail_idx) > vq->num)) {
- vq_err(vq, "Guest moved used index from %u to %u",
- last_avail_idx, vq->avail_idx);
- return -EFAULT;
- }
+ if (vq->avail_idx == vq->last_avail_idx) {
+ if (unlikely(vhost_get_avail(vq, avail_idx, &vq->avail->idx))) {
+ vq_err(vq, "Failed to access avail idx at %p\n",
+ &vq->avail->idx);
+ return -EFAULT;
+ }
+ vq->avail_idx = vhost16_to_cpu(vq, avail_idx);
- /* If there's nothing new since last we looked, return invalid. */
- if (vq->avail_idx == last_avail_idx)
- return vq->num;
+ if (unlikely((u16)(vq->avail_idx - last_avail_idx) > vq->num)) {
+ vq_err(vq, "Guest moved used index from %u to %u",
+ last_avail_idx, vq->avail_idx);
+ return -EFAULT;
+ }
+
+ /* If there's nothing new since last we looked, return
+ * invalid.
+ */
+ if (vq->avail_idx == last_avail_idx)
+ return vq->num;
- /* Only get avail ring entries after they have been exposed by guest. */
- smp_rmb();
+ /* Only get avail ring entries after they have been
+ * exposed by guest.
+ */
+ smp_rmb();
+ }
/* Grab the next descriptor number they're advertising, and increment
* the index we've seen. */
- if (unlikely(vhost_get_user(vq, ring_head,
+ if (unlikely(vhost_get_avail(vq, ring_head,
&vq->avail->ring[last_avail_idx & (vq->num - 1)]))) {
vq_err(vq, "Failed to read head: idx %d address %p\n",
last_avail_idx,
* with the barrier that the Guest executes when enabling
* interrupts. */
smp_mb();
- if (vhost_get_user(vq, flags, &vq->avail->flags)) {
+ if (vhost_get_avail(vq, flags, &vq->avail->flags)) {
vq_err(vq, "Failed to get flags");
return true;
}
* interrupts. */
smp_mb();
- if (vhost_get_user(vq, event, vhost_used_event(vq))) {
+ if (vhost_get_avail(vq, event, vhost_used_event(vq))) {
vq_err(vq, "Failed to get used event idx");
return true;
}
if (vq->avail_idx != vq->last_avail_idx)
return false;
- r = vhost_get_user(vq, avail_idx, &vq->avail->idx);
+ r = vhost_get_avail(vq, avail_idx, &vq->avail->idx);
if (unlikely(r))
return false;
vq->avail_idx = vhost16_to_cpu(vq, avail_idx);
/* They could have slipped one in as we were doing that: make
* sure it's written, then check again. */
smp_mb();
- r = vhost_get_user(vq, avail_idx, &vq->avail->idx);
+ r = vhost_get_avail(vq, avail_idx, &vq->avail->idx);
if (r) {
vq_err(vq, "Failed to check avail idx at %p: %d\n",
&vq->avail->idx, r);
int numem;
};
+enum vhost_uaddr_type {
+ VHOST_ADDR_DESC = 0,
+ VHOST_ADDR_AVAIL = 1,
+ VHOST_ADDR_USED = 2,
+ VHOST_NUM_ADDRS = 3,
+};
+
/* The virtqueue structure describes a queue attached to a device. */
struct vhost_virtqueue {
struct vhost_dev *dev;
struct vring_desc __user *desc;
struct vring_avail __user *avail;
struct vring_used __user *used;
+ const struct vhost_umem_node *meta_iotlb[VHOST_NUM_ADDRS];
struct file *kick;
struct file *call;
struct file *error;
*/
#include <linux/module.h>
+#include <linux/sched/mm.h>
#include <linux/kernel.h>
#include <linux/gpio.h>
#include <linux/platform_device.h>
#include <linux/uaccess.h>
#include <linux/platform_device.h>
#include <linux/module.h>
+#include <linux/sched/signal.h>
/*
* Cursor position address
*/
#include <linux/fb.h>
+#include <linux/nmi.h>
+
#include "nv_type.h"
#include "nv_proto.h"
#include "nv_dma.h"
#include <linux/jiffies.h>
#include <linux/module.h>
#include <linux/platform_device.h>
-#include <linux/sched.h>
+#include <linux/sched/signal.h>
#include <linux/slab.h>
#include <linux/workqueue.h>
#include <linux/of_device.h>
#include <linux/wait.h>
#include <linux/mm.h>
#include <linux/mount.h>
+#include <linux/magic.h>
/*
* Balloon device works in 4K page units. So each page is pointed to by
* optionally stat.
*/
nvqs = virtio_has_feature(vb->vdev, VIRTIO_BALLOON_F_STATS_VQ) ? 3 : 2;
- err = vb->vdev->config->find_vqs(vb->vdev, nvqs, vqs, callbacks, names);
+ err = vb->vdev->config->find_vqs(vb->vdev, nvqs, vqs, callbacks, names,
+ NULL);
if (err)
return err;
static const char * const names[] = { "events", "status" };
int err;
- err = vi->vdev->config->find_vqs(vi->vdev, 2, vqs, cbs, names);
+ err = vi->vdev->config->find_vqs(vi->vdev, 2, vqs, cbs, names,
+ NULL);
if (err)
return err;
vi->evt = vqs[0];
#include <linux/spinlock.h>
#include <linux/virtio.h>
#include <linux/virtio_config.h>
-#include <linux/virtio_mmio.h>
+#include <uapi/linux/virtio_mmio.h>
#include <linux/virtio_ring.h>
static int vm_find_vqs(struct virtio_device *vdev, unsigned nvqs,
struct virtqueue *vqs[],
vq_callback_t *callbacks[],
- const char * const names[])
+ const char * const names[],
+ struct irq_affinity *desc)
{
struct virtio_mmio_device *vm_dev = to_virtio_mmio_device(vdev);
unsigned int irq = platform_get_irq(vm_dev->pdev, 0);
struct virtio_pci_device *vp_dev = to_vp_device(vdev);
int i;
- if (vp_dev->intx_enabled)
- synchronize_irq(vp_dev->pci_dev->irq);
-
- for (i = 0; i < vp_dev->msix_vectors; ++i)
+ synchronize_irq(pci_irq_vector(vp_dev->pci_dev, 0));
+ for (i = 1; i < vp_dev->msix_vectors; i++)
synchronize_irq(pci_irq_vector(vp_dev->pci_dev, i));
}
static irqreturn_t vp_vring_interrupt(int irq, void *opaque)
{
struct virtio_pci_device *vp_dev = opaque;
- struct virtio_pci_vq_info *info;
irqreturn_t ret = IRQ_NONE;
- unsigned long flags;
+ struct virtqueue *vq;
- spin_lock_irqsave(&vp_dev->lock, flags);
- list_for_each_entry(info, &vp_dev->virtqueues, node) {
- if (vring_interrupt(irq, info->vq) == IRQ_HANDLED)
+ list_for_each_entry(vq, &vp_dev->vdev.vqs, list) {
+ if (vq->callback && vring_interrupt(irq, vq) == IRQ_HANDLED)
ret = IRQ_HANDLED;
}
- spin_unlock_irqrestore(&vp_dev->lock, flags);
return ret;
}
return vp_vring_interrupt(irq, opaque);
}
-static int vp_request_msix_vectors(struct virtio_device *vdev, int nvectors,
- bool per_vq_vectors)
+static void vp_remove_vqs(struct virtio_device *vdev)
{
struct virtio_pci_device *vp_dev = to_vp_device(vdev);
- const char *name = dev_name(&vp_dev->vdev.dev);
- unsigned i, v;
- int err = -ENOMEM;
-
- vp_dev->msix_vectors = nvectors;
-
- vp_dev->msix_names = kmalloc(nvectors * sizeof *vp_dev->msix_names,
- GFP_KERNEL);
- if (!vp_dev->msix_names)
- goto error;
- vp_dev->msix_affinity_masks
- = kzalloc(nvectors * sizeof *vp_dev->msix_affinity_masks,
- GFP_KERNEL);
- if (!vp_dev->msix_affinity_masks)
- goto error;
- for (i = 0; i < nvectors; ++i)
- if (!alloc_cpumask_var(&vp_dev->msix_affinity_masks[i],
- GFP_KERNEL))
- goto error;
-
- err = pci_alloc_irq_vectors(vp_dev->pci_dev, nvectors, nvectors,
- PCI_IRQ_MSIX);
- if (err < 0)
- goto error;
- vp_dev->msix_enabled = 1;
-
- /* Set the vector used for configuration */
- v = vp_dev->msix_used_vectors;
- snprintf(vp_dev->msix_names[v], sizeof *vp_dev->msix_names,
- "%s-config", name);
- err = request_irq(pci_irq_vector(vp_dev->pci_dev, v),
- vp_config_changed, 0, vp_dev->msix_names[v],
- vp_dev);
- if (err)
- goto error;
- ++vp_dev->msix_used_vectors;
-
- v = vp_dev->config_vector(vp_dev, v);
- /* Verify we had enough resources to assign the vector */
- if (v == VIRTIO_MSI_NO_VECTOR) {
- err = -EBUSY;
- goto error;
- }
-
- if (!per_vq_vectors) {
- /* Shared vector for all VQs */
- v = vp_dev->msix_used_vectors;
- snprintf(vp_dev->msix_names[v], sizeof *vp_dev->msix_names,
- "%s-virtqueues", name);
- err = request_irq(pci_irq_vector(vp_dev->pci_dev, v),
- vp_vring_interrupt, 0, vp_dev->msix_names[v],
- vp_dev);
- if (err)
- goto error;
- ++vp_dev->msix_used_vectors;
- }
- return 0;
-error:
- return err;
-}
-
-static struct virtqueue *vp_setup_vq(struct virtio_device *vdev, unsigned index,
- void (*callback)(struct virtqueue *vq),
- const char *name,
- u16 msix_vec)
-{
- struct virtio_pci_device *vp_dev = to_vp_device(vdev);
- struct virtio_pci_vq_info *info = kmalloc(sizeof *info, GFP_KERNEL);
- struct virtqueue *vq;
- unsigned long flags;
-
- /* fill out our structure that represents an active queue */
- if (!info)
- return ERR_PTR(-ENOMEM);
+ struct virtqueue *vq, *n;
- vq = vp_dev->setup_vq(vp_dev, info, index, callback, name, msix_vec);
- if (IS_ERR(vq))
- goto out_info;
+ list_for_each_entry_safe(vq, n, &vdev->vqs, list) {
+ if (vp_dev->msix_vector_map) {
+ int v = vp_dev->msix_vector_map[vq->index];
- info->vq = vq;
- if (callback) {
- spin_lock_irqsave(&vp_dev->lock, flags);
- list_add(&info->node, &vp_dev->virtqueues);
- spin_unlock_irqrestore(&vp_dev->lock, flags);
- } else {
- INIT_LIST_HEAD(&info->node);
+ if (v != VIRTIO_MSI_NO_VECTOR)
+ free_irq(pci_irq_vector(vp_dev->pci_dev, v),
+ vq);
+ }
+ vp_dev->del_vq(vq);
}
-
- vp_dev->vqs[index] = info;
- return vq;
-
-out_info:
- kfree(info);
- return vq;
-}
-
-static void vp_del_vq(struct virtqueue *vq)
-{
- struct virtio_pci_device *vp_dev = to_vp_device(vq->vdev);
- struct virtio_pci_vq_info *info = vp_dev->vqs[vq->index];
- unsigned long flags;
-
- spin_lock_irqsave(&vp_dev->lock, flags);
- list_del(&info->node);
- spin_unlock_irqrestore(&vp_dev->lock, flags);
-
- vp_dev->del_vq(info);
- kfree(info);
}
/* the config->del_vqs() implementation */
void vp_del_vqs(struct virtio_device *vdev)
{
struct virtio_pci_device *vp_dev = to_vp_device(vdev);
- struct virtqueue *vq, *n;
int i;
- list_for_each_entry_safe(vq, n, &vdev->vqs, list) {
- if (vp_dev->per_vq_vectors) {
- int v = vp_dev->vqs[vq->index]->msix_vector;
-
- if (v != VIRTIO_MSI_NO_VECTOR)
- free_irq(pci_irq_vector(vp_dev->pci_dev, v),
- vq);
- }
- vp_del_vq(vq);
- }
- vp_dev->per_vq_vectors = false;
-
- if (vp_dev->intx_enabled) {
- free_irq(vp_dev->pci_dev->irq, vp_dev);
- vp_dev->intx_enabled = 0;
- }
+ if (WARN_ON_ONCE(list_empty_careful(&vdev->vqs)))
+ return;
- for (i = 0; i < vp_dev->msix_used_vectors; ++i)
- free_irq(pci_irq_vector(vp_dev->pci_dev, i), vp_dev);
+ vp_remove_vqs(vdev);
- for (i = 0; i < vp_dev->msix_vectors; i++)
- if (vp_dev->msix_affinity_masks[i])
+ if (vp_dev->pci_dev->msix_enabled) {
+ for (i = 0; i < vp_dev->msix_vectors; i++)
free_cpumask_var(vp_dev->msix_affinity_masks[i]);
- if (vp_dev->msix_enabled) {
/* Disable the vector used for configuration */
vp_dev->config_vector(vp_dev, VIRTIO_MSI_NO_VECTOR);
- pci_free_irq_vectors(vp_dev->pci_dev);
- vp_dev->msix_enabled = 0;
+ kfree(vp_dev->msix_affinity_masks);
+ kfree(vp_dev->msix_names);
+ kfree(vp_dev->msix_vector_map);
}
- vp_dev->msix_vectors = 0;
- vp_dev->msix_used_vectors = 0;
- kfree(vp_dev->msix_names);
- vp_dev->msix_names = NULL;
- kfree(vp_dev->msix_affinity_masks);
- vp_dev->msix_affinity_masks = NULL;
- kfree(vp_dev->vqs);
- vp_dev->vqs = NULL;
+ free_irq(pci_irq_vector(vp_dev->pci_dev, 0), vp_dev);
+ pci_free_irq_vectors(vp_dev->pci_dev);
}
static int vp_find_vqs_msix(struct virtio_device *vdev, unsigned nvqs,
- struct virtqueue *vqs[],
- vq_callback_t *callbacks[],
- const char * const names[],
- bool per_vq_vectors)
+ struct virtqueue *vqs[], vq_callback_t *callbacks[],
+ const char * const names[], struct irq_affinity *desc)
{
struct virtio_pci_device *vp_dev = to_vp_device(vdev);
+ const char *name = dev_name(&vp_dev->vdev.dev);
+ int i, err = -ENOMEM, allocated_vectors, nvectors;
+ unsigned flags = PCI_IRQ_MSIX;
+ bool shared = false;
u16 msix_vec;
- int i, err, nvectors, allocated_vectors;
- vp_dev->vqs = kcalloc(nvqs, sizeof(*vp_dev->vqs), GFP_KERNEL);
- if (!vp_dev->vqs)
- return -ENOMEM;
+ if (desc) {
+ flags |= PCI_IRQ_AFFINITY;
+ desc->pre_vectors++; /* virtio config vector */
+ }
- if (per_vq_vectors) {
- /* Best option: one for change interrupt, one per vq. */
- nvectors = 1;
- for (i = 0; i < nvqs; ++i)
- if (callbacks[i])
- ++nvectors;
- } else {
- /* Second best: one for change, shared for all vqs. */
- nvectors = 2;
+ nvectors = 1;
+ for (i = 0; i < nvqs; i++)
+ if (callbacks[i])
+ nvectors++;
+
+ /* Try one vector per queue first. */
+ err = pci_alloc_irq_vectors_affinity(vp_dev->pci_dev, nvectors,
+ nvectors, flags, desc);
+ if (err < 0) {
+ /* Fallback to one vector for config, one shared for queues. */
+ shared = true;
+ err = pci_alloc_irq_vectors(vp_dev->pci_dev, 2, 2,
+ PCI_IRQ_MSIX);
+ if (err < 0)
+ return err;
+ }
+ if (err < 0)
+ return err;
+
+ vp_dev->msix_vectors = nvectors;
+ vp_dev->msix_names = kmalloc_array(nvectors,
+ sizeof(*vp_dev->msix_names), GFP_KERNEL);
+ if (!vp_dev->msix_names)
+ goto out_free_irq_vectors;
+
+ vp_dev->msix_affinity_masks = kcalloc(nvectors,
+ sizeof(*vp_dev->msix_affinity_masks), GFP_KERNEL);
+ if (!vp_dev->msix_affinity_masks)
+ goto out_free_msix_names;
+
+ for (i = 0; i < nvectors; ++i) {
+ if (!alloc_cpumask_var(&vp_dev->msix_affinity_masks[i],
+ GFP_KERNEL))
+ goto out_free_msix_affinity_masks;
}
- err = vp_request_msix_vectors(vdev, nvectors, per_vq_vectors);
+ /* Set the vector used for configuration */
+ snprintf(vp_dev->msix_names[0], sizeof(*vp_dev->msix_names),
+ "%s-config", name);
+ err = request_irq(pci_irq_vector(vp_dev->pci_dev, 0), vp_config_changed,
+ 0, vp_dev->msix_names[0], vp_dev);
if (err)
- goto error_find;
+ goto out_free_msix_affinity_masks;
+
+ /* Verify we had enough resources to assign the vector */
+ if (vp_dev->config_vector(vp_dev, 0) == VIRTIO_MSI_NO_VECTOR) {
+ err = -EBUSY;
+ goto out_free_config_irq;
+ }
+
+ vp_dev->msix_vector_map = kmalloc_array(nvqs,
+ sizeof(*vp_dev->msix_vector_map), GFP_KERNEL);
+ if (!vp_dev->msix_vector_map)
+ goto out_disable_config_irq;
- vp_dev->per_vq_vectors = per_vq_vectors;
- allocated_vectors = vp_dev->msix_used_vectors;
+ allocated_vectors = 1; /* vector 0 is the config interrupt */
for (i = 0; i < nvqs; ++i) {
if (!names[i]) {
vqs[i] = NULL;
continue;
}
- if (!callbacks[i])
- msix_vec = VIRTIO_MSI_NO_VECTOR;
- else if (vp_dev->per_vq_vectors)
- msix_vec = allocated_vectors++;
+ if (callbacks[i])
+ msix_vec = allocated_vectors;
else
- msix_vec = VP_MSIX_VQ_VECTOR;
- vqs[i] = vp_setup_vq(vdev, i, callbacks[i], names[i], msix_vec);
+ msix_vec = VIRTIO_MSI_NO_VECTOR;
+
+ vqs[i] = vp_dev->setup_vq(vp_dev, i, callbacks[i], names[i],
+ msix_vec);
if (IS_ERR(vqs[i])) {
err = PTR_ERR(vqs[i]);
- goto error_find;
+ goto out_remove_vqs;
}
- if (!vp_dev->per_vq_vectors || msix_vec == VIRTIO_MSI_NO_VECTOR)
+ if (msix_vec == VIRTIO_MSI_NO_VECTOR) {
+ vp_dev->msix_vector_map[i] = VIRTIO_MSI_NO_VECTOR;
continue;
+ }
- /* allocate per-vq irq if available and necessary */
- snprintf(vp_dev->msix_names[msix_vec],
- sizeof *vp_dev->msix_names,
- "%s-%s",
+ snprintf(vp_dev->msix_names[i + 1],
+ sizeof(*vp_dev->msix_names), "%s-%s",
dev_name(&vp_dev->vdev.dev), names[i]);
err = request_irq(pci_irq_vector(vp_dev->pci_dev, msix_vec),
- vring_interrupt, 0,
- vp_dev->msix_names[msix_vec],
- vqs[i]);
- if (err)
- goto error_find;
+ vring_interrupt, IRQF_SHARED,
+ vp_dev->msix_names[i + 1], vqs[i]);
+ if (err) {
+ /* don't free this irq on error */
+ vp_dev->msix_vector_map[i] = VIRTIO_MSI_NO_VECTOR;
+ goto out_remove_vqs;
+ }
+ vp_dev->msix_vector_map[i] = msix_vec;
+
+ /*
+ * Use a different vector for each queue if they are available,
+ * else share the same vector for all VQs.
+ */
+ if (!shared)
+ allocated_vectors++;
}
+
return 0;
-error_find:
- vp_del_vqs(vdev);
+out_remove_vqs:
+ vp_remove_vqs(vdev);
+ kfree(vp_dev->msix_vector_map);
+out_disable_config_irq:
+ vp_dev->config_vector(vp_dev, VIRTIO_MSI_NO_VECTOR);
+out_free_config_irq:
+ free_irq(pci_irq_vector(vp_dev->pci_dev, 0), vp_dev);
+out_free_msix_affinity_masks:
+ for (i = 0; i < nvectors; i++) {
+ if (vp_dev->msix_affinity_masks[i])
+ free_cpumask_var(vp_dev->msix_affinity_masks[i]);
+ }
+ kfree(vp_dev->msix_affinity_masks);
+out_free_msix_names:
+ kfree(vp_dev->msix_names);
+out_free_irq_vectors:
+ pci_free_irq_vectors(vp_dev->pci_dev);
return err;
}
struct virtio_pci_device *vp_dev = to_vp_device(vdev);
int i, err;
- vp_dev->vqs = kcalloc(nvqs, sizeof(*vp_dev->vqs), GFP_KERNEL);
- if (!vp_dev->vqs)
- return -ENOMEM;
-
err = request_irq(vp_dev->pci_dev->irq, vp_interrupt, IRQF_SHARED,
dev_name(&vdev->dev), vp_dev);
if (err)
- goto out_del_vqs;
+ return err;
- vp_dev->intx_enabled = 1;
- vp_dev->per_vq_vectors = false;
for (i = 0; i < nvqs; ++i) {
if (!names[i]) {
vqs[i] = NULL;
continue;
}
- vqs[i] = vp_setup_vq(vdev, i, callbacks[i], names[i],
+ vqs[i] = vp_dev->setup_vq(vp_dev, i, callbacks[i], names[i],
VIRTIO_MSI_NO_VECTOR);
if (IS_ERR(vqs[i])) {
err = PTR_ERR(vqs[i]);
- goto out_del_vqs;
+ goto out_remove_vqs;
}
}
return 0;
-out_del_vqs:
- vp_del_vqs(vdev);
+
+out_remove_vqs:
+ vp_remove_vqs(vdev);
+ free_irq(pci_irq_vector(vp_dev->pci_dev, 0), vp_dev);
return err;
}
/* the config->find_vqs() implementation */
int vp_find_vqs(struct virtio_device *vdev, unsigned nvqs,
- struct virtqueue *vqs[],
- vq_callback_t *callbacks[],
- const char * const names[])
+ struct virtqueue *vqs[], vq_callback_t *callbacks[],
+ const char * const names[], struct irq_affinity *desc)
{
int err;
- /* Try MSI-X with one vector per queue. */
- err = vp_find_vqs_msix(vdev, nvqs, vqs, callbacks, names, true);
+ err = vp_find_vqs_msix(vdev, nvqs, vqs, callbacks, names, desc);
if (!err)
return 0;
- /* Fallback: MSI-X with one vector for config, one shared for queues. */
- err = vp_find_vqs_msix(vdev, nvqs, vqs, callbacks, names, false);
- if (!err)
- return 0;
- /* Finally fall back to regular interrupts. */
return vp_find_vqs_intx(vdev, nvqs, vqs, callbacks, names);
}
{
struct virtio_device *vdev = vq->vdev;
struct virtio_pci_device *vp_dev = to_vp_device(vdev);
- struct virtio_pci_vq_info *info = vp_dev->vqs[vq->index];
- struct cpumask *mask;
- unsigned int irq;
if (!vq->callback)
return -EINVAL;
- if (vp_dev->msix_enabled) {
- mask = vp_dev->msix_affinity_masks[info->msix_vector];
- irq = pci_irq_vector(vp_dev->pci_dev, info->msix_vector);
+ if (vp_dev->pci_dev->msix_enabled) {
+ int vec = vp_dev->msix_vector_map[vq->index];
+ struct cpumask *mask = vp_dev->msix_affinity_masks[vec];
+ unsigned int irq = pci_irq_vector(vp_dev->pci_dev, vec);
+
if (cpu == -1)
irq_set_affinity_hint(irq, NULL);
else {
return 0;
}
+const struct cpumask *vp_get_vq_affinity(struct virtio_device *vdev, int index)
+{
+ struct virtio_pci_device *vp_dev = to_vp_device(vdev);
+ unsigned int *map = vp_dev->msix_vector_map;
+
+ if (!map || map[index] == VIRTIO_MSI_NO_VECTOR)
+ return NULL;
+
+ return pci_irq_get_affinity(vp_dev->pci_dev, map[index]);
+}
+
#ifdef CONFIG_PM_SLEEP
static int virtio_pci_freeze(struct device *dev)
{
vp_dev->vdev.dev.parent = &pci_dev->dev;
vp_dev->vdev.dev.release = virtio_pci_release_dev;
vp_dev->pci_dev = pci_dev;
- INIT_LIST_HEAD(&vp_dev->virtqueues);
- spin_lock_init(&vp_dev->lock);
/* enable the device */
rc = pci_enable_device(pci_dev);
#include <linux/highmem.h>
#include <linux/spinlock.h>
-struct virtio_pci_vq_info {
- /* the actual virtqueue */
- struct virtqueue *vq;
-
- /* the list node for the virtqueues list */
- struct list_head node;
-
- /* MSI-X vector (or none) */
- unsigned msix_vector;
-};
-
/* Our device structure */
struct virtio_pci_device {
struct virtio_device vdev;
/* the IO mapping for the PCI config space */
void __iomem *ioaddr;
- /* a list of queues so we can dispatch IRQs */
- spinlock_t lock;
- struct list_head virtqueues;
-
- /* array of all queues for house-keeping */
- struct virtio_pci_vq_info **vqs;
-
- /* MSI-X support */
- int msix_enabled;
- int intx_enabled;
cpumask_var_t *msix_affinity_masks;
/* Name strings for interrupts. This size should be enough,
* and I'm too lazy to allocate each name separately. */
char (*msix_names)[256];
- /* Number of available vectors */
- unsigned msix_vectors;
- /* Vectors allocated, excluding per-vq vectors if any */
- unsigned msix_used_vectors;
-
- /* Whether we have vector per vq */
- bool per_vq_vectors;
+ /* Total Number of MSI-X vectors (including per-VQ ones). */
+ int msix_vectors;
+ /* Map of per-VQ MSI-X vectors, may be NULL */
+ unsigned *msix_vector_map;
struct virtqueue *(*setup_vq)(struct virtio_pci_device *vp_dev,
- struct virtio_pci_vq_info *info,
unsigned idx,
void (*callback)(struct virtqueue *vq),
const char *name,
u16 msix_vec);
- void (*del_vq)(struct virtio_pci_vq_info *info);
+ void (*del_vq)(struct virtqueue *vq);
u16 (*config_vector)(struct virtio_pci_device *vp_dev, u16 vector);
};
-/* Constants for MSI-X */
-/* Use first vector for configuration changes, second and the rest for
- * virtqueues Thus, we need at least 2 vectors for MSI. */
-enum {
- VP_MSIX_CONFIG_VECTOR = 0,
- VP_MSIX_VQ_VECTOR = 1,
-};
-
/* Convert a generic virtio device to our structure */
static struct virtio_pci_device *to_vp_device(struct virtio_device *vdev)
{
void vp_del_vqs(struct virtio_device *vdev);
/* the config->find_vqs() implementation */
int vp_find_vqs(struct virtio_device *vdev, unsigned nvqs,
- struct virtqueue *vqs[],
- vq_callback_t *callbacks[],
- const char * const names[]);
+ struct virtqueue *vqs[], vq_callback_t *callbacks[],
+ const char * const names[], struct irq_affinity *desc);
const char *vp_bus_name(struct virtio_device *vdev);
/* Setup the affinity for a virtqueue:
*/
int vp_set_vq_affinity(struct virtqueue *vq, int cpu);
+const struct cpumask *vp_get_vq_affinity(struct virtio_device *vdev, int index);
+
#if IS_ENABLED(CONFIG_VIRTIO_PCI_LEGACY)
int virtio_pci_legacy_probe(struct virtio_pci_device *);
void virtio_pci_legacy_remove(struct virtio_pci_device *);
}
static struct virtqueue *setup_vq(struct virtio_pci_device *vp_dev,
- struct virtio_pci_vq_info *info,
unsigned index,
void (*callback)(struct virtqueue *vq),
const char *name,
if (!num || ioread32(vp_dev->ioaddr + VIRTIO_PCI_QUEUE_PFN))
return ERR_PTR(-ENOENT);
- info->msix_vector = msix_vec;
-
/* create the vring */
vq = vring_create_virtqueue(index, num,
VIRTIO_PCI_VRING_ALIGN, &vp_dev->vdev,
return ERR_PTR(err);
}
-static void del_vq(struct virtio_pci_vq_info *info)
+static void del_vq(struct virtqueue *vq)
{
- struct virtqueue *vq = info->vq;
struct virtio_pci_device *vp_dev = to_vp_device(vq->vdev);
iowrite16(vq->index, vp_dev->ioaddr + VIRTIO_PCI_QUEUE_SEL);
- if (vp_dev->msix_enabled) {
+ if (vp_dev->pci_dev->msix_enabled) {
iowrite16(VIRTIO_MSI_NO_VECTOR,
vp_dev->ioaddr + VIRTIO_MSI_QUEUE_VECTOR);
/* Flush the write out to device */
.finalize_features = vp_finalize_features,
.bus_name = vp_bus_name,
.set_vq_affinity = vp_set_vq_affinity,
+ .get_vq_affinity = vp_get_vq_affinity,
};
/* the PCI probing function */
}
static struct virtqueue *setup_vq(struct virtio_pci_device *vp_dev,
- struct virtio_pci_vq_info *info,
unsigned index,
void (*callback)(struct virtqueue *vq),
const char *name,
/* get offset of notification word for this vq */
off = vp_ioread16(&cfg->queue_notify_off);
- info->msix_vector = msix_vec;
-
/* create the vring */
vq = vring_create_virtqueue(index, num,
SMP_CACHE_BYTES, &vp_dev->vdev,
}
static int vp_modern_find_vqs(struct virtio_device *vdev, unsigned nvqs,
- struct virtqueue *vqs[],
- vq_callback_t *callbacks[],
- const char * const names[])
+ struct virtqueue *vqs[], vq_callback_t *callbacks[],
+ const char * const names[], struct irq_affinity *desc)
{
struct virtio_pci_device *vp_dev = to_vp_device(vdev);
struct virtqueue *vq;
- int rc = vp_find_vqs(vdev, nvqs, vqs, callbacks, names);
+ int rc = vp_find_vqs(vdev, nvqs, vqs, callbacks, names, desc);
if (rc)
return rc;
return 0;
}
-static void del_vq(struct virtio_pci_vq_info *info)
+static void del_vq(struct virtqueue *vq)
{
- struct virtqueue *vq = info->vq;
struct virtio_pci_device *vp_dev = to_vp_device(vq->vdev);
vp_iowrite16(vq->index, &vp_dev->common->queue_select);
- if (vp_dev->msix_enabled) {
+ if (vp_dev->pci_dev->msix_enabled) {
vp_iowrite16(VIRTIO_MSI_NO_VECTOR,
&vp_dev->common->queue_msix_vector);
/* Flush the write out to device */
.finalize_features = vp_finalize_features,
.bus_name = vp_bus_name,
.set_vq_affinity = vp_set_vq_affinity,
+ .get_vq_affinity = vp_get_vq_affinity,
};
static const struct virtio_config_ops virtio_pci_config_ops = {
.finalize_features = vp_finalize_features,
.bus_name = vp_bus_name,
.set_vq_affinity = vp_set_vq_affinity,
+ .get_vq_affinity = vp_get_vq_affinity,
};
/**
#include <linux/spinlock.h>
#include <linux/list.h>
-#include <linux/sched.h> /* schedule_timeout() */
+#include <linux/sched/signal.h>
#include <linux/delay.h>
#include <linux/export.h>
#include <linux/delay.h>
#include <linux/kthread.h>
#include <linux/slab.h>
+#include <linux/sched/signal.h>
#include <linux/export.h>
#include <linux/moduleparam.h>
config MENF21BMC_WATCHDOG
tristate "MEN 14F021P00 BMC Watchdog"
depends on MFD_MENF21BMC || COMPILE_TEST
+ depends on I2C
select WATCHDOG_CORE
help
Say Y here to include support for the MEN 14F021P00 BMC Watchdog.
config WM831X_WATCHDOG
tristate "WM831x watchdog"
- depends on MFD_WM831X || COMPILE_TEST
+ depends on MFD_WM831X
select WATCHDOG_CORE
help
Support for the watchdog in the WM831x AudioPlus PMICs. When
config ARM_SP805_WATCHDOG
tristate "ARM SP805 Watchdog"
- depends on (ARM || ARM64) && (ARM_AMBA || COMPILE_TEST)
+ depends on (ARM || ARM64 || COMPILE_TEST) && ARM_AMBA
select WATCHDOG_CORE
help
ARM Primecell SP805 Watchdog timer. This will reboot your system when
config UX500_WATCHDOG
tristate "ST-Ericsson Ux500 watchdog"
- depends on MFD_DB8500_PRCMU || (ARM && COMPILE_TEST)
+ depends on MFD_DB8500_PRCMU
select WATCHDOG_CORE
default y
help
config RETU_WATCHDOG
tristate "Retu watchdog"
- depends on MFD_RETU || COMPILE_TEST
+ depends on MFD_RETU
select WATCHDOG_CORE
help
Retu watchdog driver for Nokia Internet Tablets (770, N800,
config GEODE_WDT
tristate "AMD Geode CS5535/CS5536 Watchdog"
- depends on CS5535_MFGPT || (X86 && COMPILE_TEST)
+ depends on CS5535_MFGPT
help
This driver enables a watchdog capability built into the
CS5535/CS5536 companion chips for the AMD Geode GX and LX
config KEMPLD_WDT
tristate "Kontron COM Watchdog Timer"
- depends on MFD_KEMPLD || COMPILE_TEST
+ depends on MFD_KEMPLD
select WATCHDOG_CORE
help
Support for the PLD watchdog on some Kontron ETX and COMexpress
config BCM2835_WDT
tristate "Broadcom BCM2835 hardware watchdog"
- depends on ARCH_BCM2835 || COMPILE_TEST
+ depends on ARCH_BCM2835 || (OF && COMPILE_TEST)
select WATCHDOG_CORE
help
Watchdog driver for the built in watchdog hardware in Broadcom
unsigned int timeout)
{
struct kempld_device_data *pld = wdt_data->pld;
- u32 prescaler = kempld_prescaler[PRESCALER_21];
+ u32 prescaler;
u64 stage_timeout64;
u32 stage_timeout;
u32 remainder;
u8 stage_cfg;
+#if GCC_VERSION < 40400
+ /* work around a bug compiling do_div() */
+ prescaler = READ_ONCE(kempld_prescaler[PRESCALER_21]);
+#else
+ prescaler = kempld_prescaler[PRESCALER_21];
+#endif
+
if (!stage)
return -EINVAL;
#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
+#include <linux/hrtimer.h>
#include <linux/init.h>
-#include <linux/jiffies.h>
#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/moduleparam.h>
#include <linux/reboot.h>
-#include <linux/timer.h>
#include <linux/types.h>
#include <linux/watchdog.h>
MODULE_PARM_DESC(soft_panic,
"Softdog action, set to 1 to panic, 0 to reboot (default=0)");
-static void softdog_fire(unsigned long data)
+static struct hrtimer softdog_ticktock;
+static struct hrtimer softdog_preticktock;
+
+static enum hrtimer_restart softdog_fire(struct hrtimer *timer)
{
module_put(THIS_MODULE);
if (soft_noboot) {
emergency_restart();
pr_crit("Reboot didn't ?????\n");
}
-}
-static struct timer_list softdog_ticktock =
- TIMER_INITIALIZER(softdog_fire, 0, 0);
+ return HRTIMER_NORESTART;
+}
static struct watchdog_device softdog_dev;
-static void softdog_pretimeout(unsigned long data)
+static enum hrtimer_restart softdog_pretimeout(struct hrtimer *timer)
{
watchdog_notify_pretimeout(&softdog_dev);
-}
-static struct timer_list softdog_preticktock =
- TIMER_INITIALIZER(softdog_pretimeout, 0, 0);
+ return HRTIMER_NORESTART;
+}
static int softdog_ping(struct watchdog_device *w)
{
- if (!mod_timer(&softdog_ticktock, jiffies + (w->timeout * HZ)))
+ if (!hrtimer_active(&softdog_ticktock))
__module_get(THIS_MODULE);
+ hrtimer_start(&softdog_ticktock, ktime_set(w->timeout, 0),
+ HRTIMER_MODE_REL);
if (IS_ENABLED(CONFIG_SOFT_WATCHDOG_PRETIMEOUT)) {
if (w->pretimeout)
- mod_timer(&softdog_preticktock, jiffies +
- (w->timeout - w->pretimeout) * HZ);
+ hrtimer_start(&softdog_preticktock,
+ ktime_set(w->timeout - w->pretimeout, 0),
+ HRTIMER_MODE_REL);
else
- del_timer(&softdog_preticktock);
+ hrtimer_cancel(&softdog_preticktock);
}
return 0;
static int softdog_stop(struct watchdog_device *w)
{
- if (del_timer(&softdog_ticktock))
+ if (hrtimer_cancel(&softdog_ticktock))
module_put(THIS_MODULE);
if (IS_ENABLED(CONFIG_SOFT_WATCHDOG_PRETIMEOUT))
- del_timer(&softdog_preticktock);
+ hrtimer_cancel(&softdog_preticktock);
return 0;
}
watchdog_set_nowayout(&softdog_dev, nowayout);
watchdog_stop_on_reboot(&softdog_dev);
- if (IS_ENABLED(CONFIG_SOFT_WATCHDOG_PRETIMEOUT))
+ hrtimer_init(&softdog_ticktock, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
+ softdog_ticktock.function = softdog_fire;
+
+ if (IS_ENABLED(CONFIG_SOFT_WATCHDOG_PRETIMEOUT)) {
softdog_info.options |= WDIOF_PRETIMEOUT;
+ hrtimer_init(&softdog_preticktock, CLOCK_MONOTONIC,
+ HRTIMER_MODE_REL);
+ softdog_preticktock.function = softdog_pretimeout;
+ }
ret = watchdog_register_device(&softdog_dev);
if (ret)
#include <linux/cpu.h>
#include <linux/kernel.h>
#include <linux/sched.h>
+#include <linux/cred.h>
#include <linux/errno.h>
#include <linux/mm.h>
#include <linux/bootmem.h>
#include <linux/types.h>
#include <linux/uaccess.h>
#include <linux/sched.h>
+#include <linux/sched/mm.h>
#include <linux/spinlock.h>
#include <linux/slab.h>
#include <linux/highmem.h>
return 0;
}
EXPORT_SYMBOL_GPL(xen_swiotlb_set_dma_mask);
+
+/*
+ * Create userspace mapping for the DMA-coherent memory.
+ * This function should be called with the pages from the current domain only,
+ * passing pages mapped from other domains would lead to memory corruption.
+ */
+int
+xen_swiotlb_dma_mmap(struct device *dev, struct vm_area_struct *vma,
+ void *cpu_addr, dma_addr_t dma_addr, size_t size,
+ unsigned long attrs)
+{
+#if defined(CONFIG_ARM) || defined(CONFIG_ARM64)
+ if (__generic_dma_ops(dev)->mmap)
+ return __generic_dma_ops(dev)->mmap(dev, vma, cpu_addr,
+ dma_addr, size, attrs);
+#endif
+ return dma_common_mmap(dev, vma, cpu_addr, dma_addr, size);
+}
+EXPORT_SYMBOL_GPL(xen_swiotlb_dma_mmap);
+
+/*
+ * This function should be called with the pages from the current domain only,
+ * passing pages mapped from other domains would lead to memory corruption.
+ */
+int
+xen_swiotlb_get_sgtable(struct device *dev, struct sg_table *sgt,
+ void *cpu_addr, dma_addr_t handle, size_t size,
+ unsigned long attrs)
+{
+#if defined(CONFIG_ARM) || defined(CONFIG_ARM64)
+ if (__generic_dma_ops(dev)->get_sgtable) {
+#if 0
+ /*
+ * This check verifies that the page belongs to the current domain and
+ * is not one mapped from another domain.
+ * This check is for debug only, and should not go to production build
+ */
+ unsigned long bfn = PHYS_PFN(dma_to_phys(dev, handle));
+ BUG_ON (!page_is_ram(bfn));
+#endif
+ return __generic_dma_ops(dev)->get_sgtable(dev, sgt, cpu_addr,
+ handle, size, attrs);
+ }
+#endif
+ return dma_common_get_sgtable(dev, sgt, cpu_addr, handle, size);
+}
+EXPORT_SYMBOL_GPL(xen_swiotlb_get_sgtable);
#include <linux/string.h>
#include <linux/slab.h>
#include <linux/miscdevice.h>
-#include <linux/init.h>
#include <xen/xenbus.h>
#include <xen/xen.h>
* dentry names.
*/
static int build_path_from_dentry(struct v9fs_session_info *v9ses,
- struct dentry *dentry, char ***names)
+ struct dentry *dentry, const unsigned char ***names)
{
int n = 0, i;
- char **wnames;
+ const unsigned char **wnames;
struct dentry *ds;
for (ds = dentry; !IS_ROOT(ds); ds = ds->d_parent)
goto err_out;
for (ds = dentry, i = (n-1); i >= 0; i--, ds = ds->d_parent)
- wnames[i] = (char *)ds->d_name.name;
+ wnames[i] = ds->d_name.name;
*names = wnames;
return n;
kuid_t uid, int any)
{
struct dentry *ds;
- char **wnames, *uname;
+ const unsigned char **wnames, *uname;
int i, n, l, clone, access;
struct v9fs_session_info *v9ses;
struct p9_fid *fid, *old_fid = NULL;
fid = v9fs_fid_find(ds, uid, any);
if (fid) {
/* Found the parent fid do a lookup with that */
- fid = p9_client_walk(fid, 1, (char **)&dentry->d_name.name, 1);
+ fid = p9_client_walk(fid, 1, &dentry->d_name.name, 1);
goto fid_out;
}
up_read(&v9ses->rename_sem);
#include <linux/errno.h>
#include <linux/fs.h>
#include <linux/sched.h>
+#include <linux/cred.h>
#include <linux/parser.h>
#include <linux/idr.h>
#include <linux/slab.h>
struct dentry *dentry, char *extension, u32 perm, u8 mode)
{
int err;
- char *name;
+ const unsigned char *name;
struct p9_fid *dfid, *ofid, *fid;
struct inode *inode;
err = 0;
ofid = NULL;
fid = NULL;
- name = (char *) dentry->d_name.name;
+ name = dentry->d_name.name;
dfid = v9fs_parent_fid(dentry);
if (IS_ERR(dfid)) {
err = PTR_ERR(dfid);
struct v9fs_session_info *v9ses;
struct p9_fid *dfid, *fid;
struct inode *inode;
- char *name;
+ const unsigned char *name;
p9_debug(P9_DEBUG_VFS, "dir: %p dentry: (%pd) %p flags: %x\n",
dir, dentry, dentry, flags);
if (IS_ERR(dfid))
return ERR_CAST(dfid);
- name = (char *) dentry->d_name.name;
+ name = dentry->d_name.name;
fid = p9_client_walk(dfid, 1, &name, 1);
if (IS_ERR(fid)) {
if (fid == ERR_PTR(-ENOENT)) {
}
v9fs_blank_wstat(&wstat);
wstat.muid = v9ses->uname;
- wstat.name = (char *) new_dentry->d_name.name;
+ wstat.name = new_dentry->d_name.name;
retval = p9_client_wstat(oldfid, &wstat);
clunk_newdir:
/**
* v9fs_vfs_getattr - retrieve file metadata
- * @mnt: mount information
- * @dentry: file to get attributes on
+ * @path: Object to query
* @stat: metadata structure to populate
+ * @request_mask: Mask of STATX_xxx flags indicating the caller's interests
+ * @flags: AT_STATX_xxx setting
*
*/
static int
-v9fs_vfs_getattr(struct vfsmount *mnt, struct dentry *dentry,
- struct kstat *stat)
+v9fs_vfs_getattr(const struct path *path, struct kstat *stat,
+ u32 request_mask, unsigned int flags)
{
+ struct dentry *dentry = path->dentry;
struct v9fs_session_info *v9ses;
struct p9_fid *fid;
struct p9_wstat *st;
int err = 0;
kgid_t gid;
umode_t mode;
- char *name = NULL;
+ const unsigned char *name = NULL;
struct p9_qid qid;
struct inode *inode;
struct p9_fid *fid = NULL;
v9ses = v9fs_inode2v9ses(dir);
- name = (char *) dentry->d_name.name;
+ name = dentry->d_name.name;
p9_debug(P9_DEBUG_VFS, "name:%s flags:0x%x mode:0x%hx\n",
name, flags, omode);
struct v9fs_session_info *v9ses;
struct p9_fid *fid = NULL, *dfid = NULL;
kgid_t gid;
- char *name;
+ const unsigned char *name;
umode_t mode;
struct inode *inode;
struct p9_qid qid;
err);
goto error;
}
- name = (char *) dentry->d_name.name;
+ name = dentry->d_name.name;
err = p9_client_mkdir_dotl(dfid, name, mode, gid, &qid);
if (err < 0)
goto error;
}
static int
-v9fs_vfs_getattr_dotl(struct vfsmount *mnt, struct dentry *dentry,
- struct kstat *stat)
+v9fs_vfs_getattr_dotl(const struct path *path, struct kstat *stat,
+ u32 request_mask, unsigned int flags)
{
+ struct dentry *dentry = path->dentry;
struct v9fs_session_info *v9ses;
struct p9_fid *fid;
struct p9_stat_dotl *st;
{
int err;
kgid_t gid;
- char *name;
+ const unsigned char *name;
struct p9_qid qid;
struct inode *inode;
struct p9_fid *dfid;
struct p9_fid *fid = NULL;
struct v9fs_session_info *v9ses;
- name = (char *) dentry->d_name.name;
+ name = dentry->d_name.name;
p9_debug(P9_DEBUG_VFS, "%lu,%s,%s\n", dir->i_ino, name, symname);
v9ses = v9fs_inode2v9ses(dir);
gid = v9fs_get_fsgid_for_create(dir);
/* Server doesn't alter fid on TSYMLINK. Hence no need to clone it. */
- err = p9_client_symlink(dfid, name, (char *)symname, gid, &qid);
+ err = p9_client_symlink(dfid, name, symname, gid, &qid);
if (err < 0) {
p9_debug(P9_DEBUG_VFS, "p9_client_symlink failed %d\n", err);
if (IS_ERR(oldfid))
return PTR_ERR(oldfid);
- err = p9_client_link(dfid, oldfid, (char *)dentry->d_name.name);
+ err = p9_client_link(dfid, oldfid, dentry->d_name.name);
if (err < 0) {
p9_debug(P9_DEBUG_VFS, "p9_client_link failed %d\n", err);
{
int err;
kgid_t gid;
- char *name;
+ const unsigned char *name;
umode_t mode;
struct v9fs_session_info *v9ses;
struct p9_fid *fid = NULL, *dfid = NULL;
err);
goto error;
}
- name = (char *) dentry->d_name.name;
+ name = dentry->d_name.name;
err = p9_client_mknod_dotl(dfid, name, mode, rdev, gid, &qid);
if (err < 0)
* (C) 1991 Linus Torvalds - minix filesystem
*/
#include <linux/sched.h>
+#include <linux/cred.h>
#include <linux/gfp.h>
#include "affs.h"
#include <linux/parser.h>
#include <linux/magic.h>
#include <linux/sched.h>
+#include <linux/cred.h>
#include <linux/slab.h>
#include <linux/writeback.h>
#include <linux/blkdev.h>
/*
* read the attributes of an inode
*/
-int afs_getattr(struct vfsmount *mnt, struct dentry *dentry,
- struct kstat *stat)
+int afs_getattr(const struct path *path, struct kstat *stat,
+ u32 request_mask, unsigned int query_flags)
{
- struct inode *inode;
-
- inode = d_inode(dentry);
+ struct inode *inode = d_inode(path->dentry);
_enter("{ ino=%lu v=%u }", inode->i_ino, inode->i_generation);
struct afs_callback *);
extern void afs_zap_data(struct afs_vnode *);
extern int afs_validate(struct afs_vnode *, struct key *);
-extern int afs_getattr(struct vfsmount *, struct dentry *, struct kstat *);
+extern int afs_getattr(const struct path *, struct kstat *, u32, unsigned int);
extern int afs_setattr(struct dentry *, struct iattr *);
extern void afs_evict_inode(struct inode *);
extern int afs_drop_inode(struct inode *);
*/
#include <linux/slab.h>
+#include <linux/sched/signal.h>
+
#include <net/sock.h>
#include <net/af_rxrpc.h>
#include <rxrpc/packet.h>
/*
* attach the data from a bunch of pages on an inode to a call
*/
-static int afs_send_pages(struct afs_call *call, struct msghdr *msg,
- struct kvec *iov)
+static int afs_send_pages(struct afs_call *call, struct msghdr *msg)
{
struct page *pages[8];
unsigned count, n, loop, offset, to;
loop = 0;
do {
+ struct bio_vec bvec = {.bv_page = pages[loop],
+ .bv_offset = offset};
msg->msg_flags = 0;
to = PAGE_SIZE;
if (first + loop >= last)
to = call->last_to;
else
msg->msg_flags = MSG_MORE;
- iov->iov_base = kmap(pages[loop]) + offset;
- iov->iov_len = to - offset;
+ bvec.bv_len = to - offset;
offset = 0;
_debug("- range %u-%u%s",
offset, to, msg->msg_flags ? " [more]" : "");
- iov_iter_kvec(&msg->msg_iter, WRITE | ITER_KVEC,
- iov, 1, to - offset);
+ iov_iter_bvec(&msg->msg_iter, WRITE | ITER_BVEC,
+ &bvec, 1, to - offset);
/* have to change the state *before* sending the last
* packet as RxRPC might give us the reply before it
call->state = AFS_CALL_AWAIT_REPLY;
ret = rxrpc_kernel_send_data(afs_socket, call->rxcall,
msg, to - offset);
- kunmap(pages[loop]);
if (ret < 0)
break;
} while (++loop < count);
goto error_do_abort;
if (call->send_pages) {
- ret = afs_send_pages(call, &msg, iov);
+ ret = afs_send_pages(call, &msg);
if (ret < 0)
goto error_do_abort;
}
#include <linux/backing-dev.h>
#include <linux/uio.h>
-#include <linux/sched.h>
+#include <linux/sched/signal.h>
#include <linux/fs.h>
#include <linux/file.h>
#include <linux/mm.h>
return ret;
ret = rw_verify_area(READ, file, &req->ki_pos, iov_iter_count(&iter));
if (!ret)
- ret = aio_ret(req, file->f_op->read_iter(req, &iter));
+ ret = aio_ret(req, call_read_iter(file, req, &iter));
kfree(iovec);
return ret;
}
if (!ret) {
req->ki_flags |= IOCB_WRITE;
file_start_write(file);
- ret = aio_ret(req, file->f_op->write_iter(req, &iter));
+ ret = aio_ret(req, call_write_iter(file, req, &iter));
/*
* We release freeze protection in aio_complete(). Fool lockdep
* by telling it the lock got released so that it doesn't
#include <linux/time.h>
#include <linux/mm.h>
#include <linux/string.h>
+#include <linux/sched/signal.h>
#include <linux/capability.h>
#include <linux/fsnotify.h>
#include <linux/fcntl.h>
#include <linux/mutex.h>
#include <linux/spinlock.h>
#include <linux/list.h>
+#include <linux/completion.h>
/* This is the range of ioctl() numbers we claim as ours */
#define AUTOFS_IOC_FIRST AUTOFS_IOC_READY
#include <linux/file.h>
#include <linux/fdtable.h>
#include <linux/sched.h>
+#include <linux/cred.h>
#include <linux/compat.h>
#include <linux/syscalls.h>
#include <linux/magic.h>
#include <linux/slab.h>
#include <linux/time.h>
#include <linux/signal.h>
+#include <linux/sched/signal.h>
#include <linux/file.h>
#include "autofs_i.h"
return -EIO;
}
-static int bad_inode_getattr(struct vfsmount *mnt, struct dentry *dentry,
- struct kstat *stat)
+static int bad_inode_getattr(const struct path *path, struct kstat *stat,
+ u32 request_mask, unsigned int query_flags)
{
return -EIO;
}
#include <linux/parser.h>
#include <linux/namei.h>
#include <linux/sched.h>
+#include <linux/cred.h>
#include <linux/exportfs.h>
#include "befs.h"
#include <linux/init.h>
#include <linux/coredump.h>
#include <linux/slab.h>
+#include <linux/sched/task_stack.h>
#include <linux/uaccess.h>
#include <asm/cacheflush.h>
#include <linux/utsname.h>
#include <linux/coredump.h>
#include <linux/sched.h>
+#include <linux/sched/coredump.h>
+#include <linux/sched/task_stack.h>
+#include <linux/sched/cputime.h>
+#include <linux/cred.h>
#include <linux/dax.h>
#include <linux/uaccess.h>
#include <asm/param.h>
#include <linux/fs.h>
#include <linux/stat.h>
#include <linux/sched.h>
+#include <linux/sched/coredump.h>
+#include <linux/sched/task_stack.h>
+#include <linux/sched/cputime.h>
#include <linux/mm.h>
#include <linux/mman.h>
#include <linux/errno.h>
#include <linux/kernel.h>
#include <linux/sched.h>
+#include <linux/sched/task_stack.h>
#include <linux/mm.h>
#include <linux/mman.h>
#include <linux/errno.h>
#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/init.h>
-#include <linux/sched.h>
+#include <linux/sched/mm.h>
#include <linux/magic.h>
#include <linux/binfmts.h>
#include <linux/slab.h>
#ifdef CONFIG_SYSFS
INIT_LIST_HEAD(&bdev->bd_holder_disks);
#endif
+ bdev->bd_bdi = &noop_backing_dev_info;
inode_init_once(&ei->vfs_inode);
/* Initialize mutex for freeze. */
mutex_init(&bdev->bd_fsfreeze_mutex);
spin_lock(&bdev_lock);
list_del_init(&bdev->bd_list);
spin_unlock(&bdev_lock);
- if (bdev->bd_bdi != &noop_backing_dev_info)
+ if (bdev->bd_bdi != &noop_backing_dev_info) {
bdi_put(bdev->bd_bdi);
+ bdev->bd_bdi = &noop_backing_dev_info;
+ }
}
static const struct super_operations bdev_sops = {
bdev->bd_contains = NULL;
bdev->bd_super = NULL;
bdev->bd_inode = inode;
- bdev->bd_bdi = &noop_backing_dev_info;
bdev->bd_block_size = i_blocksize(inode);
bdev->bd_part_count = 0;
bdev->bd_invalidated = 0;
return ino;
}
-static inline void btrfs_i_size_write(struct inode *inode, u64 size)
+static inline void btrfs_i_size_write(struct btrfs_inode *inode, u64 size)
{
- i_size_write(inode, size);
- BTRFS_I(inode)->disk_i_size = size;
+ i_size_write(&inode->vfs_inode, size);
+ inode->disk_i_size = size;
}
-static inline bool btrfs_is_free_space_inode(struct inode *inode)
+static inline bool btrfs_is_free_space_inode(struct btrfs_inode *inode)
{
- struct btrfs_root *root = BTRFS_I(inode)->root;
+ struct btrfs_root *root = inode->root;
if (root == root->fs_info->tree_root &&
- btrfs_ino(BTRFS_I(inode)) != BTRFS_BTREE_INODE_OBJECTID)
+ btrfs_ino(inode) != BTRFS_BTREE_INODE_OBJECTID)
return true;
- if (BTRFS_I(inode)->location.objectid == BTRFS_FREE_INO_OBJECTID)
+ if (inode->location.objectid == BTRFS_FREE_INO_OBJECTID)
return true;
return false;
}
* to grab i_mutex. It is used to avoid the endless truncate due to
* nonlocked dio read.
*/
-static inline void btrfs_inode_block_unlocked_dio(struct inode *inode)
+static inline void btrfs_inode_block_unlocked_dio(struct btrfs_inode *inode)
{
- set_bit(BTRFS_INODE_READDIO_NEED_LOCK, &BTRFS_I(inode)->runtime_flags);
+ set_bit(BTRFS_INODE_READDIO_NEED_LOCK, &inode->runtime_flags);
smp_mb();
}
-static inline void btrfs_inode_resume_unlocked_dio(struct inode *inode)
+static inline void btrfs_inode_resume_unlocked_dio(struct btrfs_inode *inode)
{
smp_mb__before_atomic();
- clear_bit(BTRFS_INODE_READDIO_NEED_LOCK,
- &BTRFS_I(inode)->runtime_flags);
+ clear_bit(BTRFS_INODE_READDIO_NEED_LOCK, &inode->runtime_flags);
}
-static inline void btrfs_print_data_csum_error(struct inode *inode,
+static inline void btrfs_print_data_csum_error(struct btrfs_inode *inode,
u64 logical_start, u32 csum, u32 csum_expected, int mirror_num)
{
- struct btrfs_root *root = BTRFS_I(inode)->root;
+ struct btrfs_root *root = inode->root;
/* Output minus objectid, which is more meaningful */
if (root->objectid >= BTRFS_LAST_FREE_OBJECTID)
btrfs_warn_rl(root->fs_info,
"csum failed root %lld ino %lld off %llu csum 0x%08x expected csum 0x%08x mirror %d",
- root->objectid, btrfs_ino(BTRFS_I(inode)),
+ root->objectid, btrfs_ino(inode),
logical_start, csum, csum_expected, mirror_num);
else
btrfs_warn_rl(root->fs_info,
"csum failed root %llu ino %llu off %llu csum 0x%08x expected csum 0x%08x mirror %d",
- root->objectid, btrfs_ino(BTRFS_I(inode)),
+ root->objectid, btrfs_ino(inode),
logical_start, csum, csum_expected, mirror_num);
}
return btrfs_bio_alloc(bdev, first_byte >> 9, BIO_MAX_PAGES, gfp_flags);
}
-static int check_compressed_csum(struct inode *inode,
+static int check_compressed_csum(struct btrfs_inode *inode,
struct compressed_bio *cb,
u64 disk_start)
{
u32 csum;
u32 *cb_sum = &cb->sums;
- if (BTRFS_I(inode)->flags & BTRFS_INODE_NODATASUM)
+ if (inode->flags & BTRFS_INODE_NODATASUM)
return 0;
for (i = 0; i < cb->nr_pages; i++) {
if (csum != *cb_sum) {
btrfs_print_data_csum_error(inode, disk_start, csum,
- *cb_sum, cb->mirror_num);
+ *cb_sum, cb->mirror_num);
ret = -EIO;
goto fail;
}
goto out;
inode = cb->inode;
- ret = check_compressed_csum(inode, cb,
+ ret = check_compressed_csum(BTRFS_I(inode), cb,
(u64)bio->bi_iter.bi_sector << 9);
if (ret)
goto csum_failed;
}
/*
- * given an address space and start/len, compress the bytes.
+ * Given an address space and start and length, compress the bytes into @pages
+ * that are allocated on demand.
*
- * pages are allocated to hold the compressed result and stored
- * in 'pages'
+ * @out_pages is an in/out parameter, holds maximum number of pages to allocate
+ * and returns number of actually allocated pages
*
- * out_pages is used to return the number of pages allocated. There
- * may be pages allocated even if we return an error
- *
- * total_in is used to return the number of bytes actually read. It
- * may be smaller then len if we had to exit early because we
+ * @total_in is used to return the number of bytes actually read. It
+ * may be smaller than the input length if we had to exit early because we
* ran out of room in the pages array or because we cross the
* max_out threshold.
*
- * total_out is used to return the total number of compressed bytes
+ * @total_out is an in/out parameter, must be set to the input length and will
+ * be also used to return the total number of compressed bytes
*
- * max_out tells us the max number of bytes that we're allowed to
+ * @max_out tells us the max number of bytes that we're allowed to
* stuff into pages
*/
int btrfs_compress_pages(int type, struct address_space *mapping,
- u64 start, unsigned long len,
- struct page **pages,
- unsigned long nr_dest_pages,
+ u64 start, struct page **pages,
unsigned long *out_pages,
unsigned long *total_in,
- unsigned long *total_out,
- unsigned long max_out)
+ unsigned long *total_out)
{
struct list_head *workspace;
int ret;
workspace = find_workspace(type);
ret = btrfs_compress_op[type-1]->compress_pages(workspace, mapping,
- start, len, pages,
- nr_dest_pages, out_pages,
- total_in, total_out,
- max_out);
+ start, pages,
+ out_pages,
+ total_in, total_out);
free_workspace(type, workspace);
return ret;
}
*
* total_out is the last byte of the buffer
*/
-int btrfs_decompress_buf2page(char *buf, unsigned long buf_start,
+int btrfs_decompress_buf2page(const char *buf, unsigned long buf_start,
unsigned long total_out, u64 disk_start,
struct bio *bio)
{
#ifndef __BTRFS_COMPRESSION_
#define __BTRFS_COMPRESSION_
+/*
+ * We want to make sure that amount of RAM required to uncompress an extent is
+ * reasonable, so we limit the total size in ram of a compressed extent to
+ * 128k. This is a crucial number because it also controls how easily we can
+ * spread reads across cpus for decompression.
+ *
+ * We also want to make sure the amount of IO required to do a random read is
+ * reasonably small, so we limit the size of a compressed extent to 128k.
+ */
+
+/* Maximum length of compressed data stored on disk */
+#define BTRFS_MAX_COMPRESSED (SZ_128K)
+/* Maximum size of data before compression */
+#define BTRFS_MAX_UNCOMPRESSED (SZ_128K)
+
void btrfs_init_compress(void);
void btrfs_exit_compress(void);
int btrfs_compress_pages(int type, struct address_space *mapping,
- u64 start, unsigned long len,
- struct page **pages,
- unsigned long nr_dest_pages,
+ u64 start, struct page **pages,
unsigned long *out_pages,
unsigned long *total_in,
- unsigned long *total_out,
- unsigned long max_out);
+ unsigned long *total_out);
int btrfs_decompress(int type, unsigned char *data_in, struct page *dest_page,
unsigned long start_byte, size_t srclen, size_t destlen);
-int btrfs_decompress_buf2page(char *buf, unsigned long buf_start,
+int btrfs_decompress_buf2page(const char *buf, unsigned long buf_start,
unsigned long total_out, u64 disk_start,
struct bio *bio);
int (*compress_pages)(struct list_head *workspace,
struct address_space *mapping,
- u64 start, unsigned long len,
+ u64 start,
struct page **pages,
- unsigned long nr_dest_pages,
unsigned long *out_pages,
unsigned long *total_in,
- unsigned long *total_out,
- unsigned long max_out);
+ unsigned long *total_out);
int (*decompress_bio)(struct list_head *workspace,
struct page **pages_in,
struct rb_node *parent = NULL;
struct tree_mod_elem *cur;
- BUG_ON(!tm);
-
tm->seq = btrfs_inc_tree_mod_seq(fs_info);
tm_root = &fs_info->tree_mod_log;
/* try to push all the items before our slot into the next leaf */
slot = path->slots[0];
+ space_needed = data_size;
+ if (slot > 0)
+ space_needed -= btrfs_leaf_free_space(fs_info, path->nodes[0]);
ret = push_leaf_left(trans, root, path, 1, space_needed, 0, slot);
if (ret < 0)
return ret;
if (wret < 0)
return wret;
if (wret) {
+ space_needed = data_size;
+ if (slot > 0)
+ space_needed -= btrfs_leaf_free_space(fs_info,
+ l);
wret = push_leaf_left(trans, root, path, space_needed,
space_needed, 0, (u32)-1);
if (wret < 0)
#define __BTRFS_CTREE__
#include <linux/mm.h>
+#include <linux/sched/signal.h>
#include <linux/highmem.h>
#include <linux/fs.h>
#include <linux/rwsem.h>
};
int btrfs_check_data_free_space(struct inode *inode, u64 start, u64 len);
-int btrfs_alloc_data_chunk_ondemand(struct inode *inode, u64 bytes);
+int btrfs_alloc_data_chunk_ondemand(struct btrfs_inode *inode, u64 bytes);
void btrfs_free_reserved_data_space(struct inode *inode, u64 start, u64 len);
void btrfs_free_reserved_data_space_noquota(struct inode *inode, u64 start,
u64 len);
struct btrfs_fs_info *fs_info);
void btrfs_trans_release_chunk_metadata(struct btrfs_trans_handle *trans);
int btrfs_orphan_reserve_metadata(struct btrfs_trans_handle *trans,
- struct inode *inode);
-void btrfs_orphan_release_metadata(struct inode *inode);
+ struct btrfs_inode *inode);
+void btrfs_orphan_release_metadata(struct btrfs_inode *inode);
int btrfs_subvolume_reserve_metadata(struct btrfs_root *root,
struct btrfs_block_rsv *rsv,
int nitems,
u64 *qgroup_reserved, bool use_global_rsv);
void btrfs_subvolume_release_metadata(struct btrfs_fs_info *fs_info,
struct btrfs_block_rsv *rsv);
-int btrfs_delalloc_reserve_metadata(struct inode *inode, u64 num_bytes);
-void btrfs_delalloc_release_metadata(struct inode *inode, u64 num_bytes);
+int btrfs_delalloc_reserve_metadata(struct btrfs_inode *inode, u64 num_bytes);
+void btrfs_delalloc_release_metadata(struct btrfs_inode *inode, u64 num_bytes);
int btrfs_delalloc_reserve_space(struct inode *inode, u64 start, u64 len);
void btrfs_delalloc_release_space(struct inode *inode, u64 start, u64 len);
void btrfs_init_block_rsv(struct btrfs_block_rsv *rsv, unsigned short type);
const char *name, int name_len);
int btrfs_insert_dir_item(struct btrfs_trans_handle *trans,
struct btrfs_root *root, const char *name,
- int name_len, struct inode *dir,
+ int name_len, struct btrfs_inode *dir,
struct btrfs_key *location, u8 type, u64 index);
struct btrfs_dir_item *btrfs_lookup_dir_item(struct btrfs_trans_handle *trans,
struct btrfs_root *root,
u64 file_start, int contig);
int btrfs_lookup_csums_range(struct btrfs_root *root, u64 start, u64 end,
struct list_head *list, int search_commit);
-void btrfs_extent_item_to_extent_map(struct inode *inode,
+void btrfs_extent_item_to_extent_map(struct btrfs_inode *inode,
const struct btrfs_path *path,
struct btrfs_file_extent_item *fi,
const bool new_inline,
int delay_iput);
void btrfs_wait_and_free_delalloc_work(struct btrfs_delalloc_work *work);
-struct extent_map *btrfs_get_extent_fiemap(struct inode *inode, struct page *page,
- size_t pg_offset, u64 start, u64 len,
- int create);
+struct extent_map *btrfs_get_extent_fiemap(struct btrfs_inode *inode,
+ struct page *page, size_t pg_offset, u64 start,
+ u64 len, int create);
noinline int can_nocow_extent(struct inode *inode, u64 offset, u64 *len,
u64 *orig_start, u64 *orig_block_len,
u64 *ram_bytes);
}
struct inode *btrfs_lookup_dentry(struct inode *dir, struct dentry *dentry);
-int btrfs_set_inode_index(struct inode *dir, u64 *index);
+int btrfs_set_inode_index(struct btrfs_inode *dir, u64 *index);
int btrfs_unlink_inode(struct btrfs_trans_handle *trans,
struct btrfs_root *root,
struct btrfs_inode *dir, struct btrfs_inode *inode,
const char *name, int name_len);
int btrfs_add_link(struct btrfs_trans_handle *trans,
- struct inode *parent_inode, struct inode *inode,
+ struct btrfs_inode *parent_inode, struct btrfs_inode *inode,
const char *name, int name_len, int add_backref, u64 index);
int btrfs_unlink_subvol(struct btrfs_trans_handle *trans,
struct btrfs_root *root,
long btrfs_ioctl_trans_end(struct file *file);
struct inode *btrfs_iget(struct super_block *s, struct btrfs_key *location,
struct btrfs_root *root, int *was_new);
-struct extent_map *btrfs_get_extent(struct inode *inode, struct page *page,
- size_t pg_offset, u64 start, u64 end,
- int create);
+struct extent_map *btrfs_get_extent(struct btrfs_inode *inode,
+ struct page *page, size_t pg_offset,
+ u64 start, u64 end, int create);
int btrfs_update_inode(struct btrfs_trans_handle *trans,
struct btrfs_root *root,
struct inode *inode);
int btrfs_update_inode_fallback(struct btrfs_trans_handle *trans,
struct btrfs_root *root, struct inode *inode);
-int btrfs_orphan_add(struct btrfs_trans_handle *trans, struct inode *inode);
+int btrfs_orphan_add(struct btrfs_trans_handle *trans,
+ struct btrfs_inode *inode);
int btrfs_orphan_cleanup(struct btrfs_root *root);
void btrfs_orphan_commit_root(struct btrfs_trans_handle *trans,
struct btrfs_root *root);
int btrfs_auto_defrag_init(void);
void btrfs_auto_defrag_exit(void);
int btrfs_add_inode_defrag(struct btrfs_trans_handle *trans,
- struct inode *inode);
+ struct btrfs_inode *inode);
int btrfs_run_defrag_inodes(struct btrfs_fs_info *fs_info);
void btrfs_cleanup_defrag_inodes(struct btrfs_fs_info *fs_info);
int btrfs_sync_file(struct file *file, loff_t start, loff_t end, int datasync);
-void btrfs_drop_extent_cache(struct inode *inode, u64 start, u64 end,
+void btrfs_drop_extent_cache(struct btrfs_inode *inode, u64 start, u64 end,
int skip_pinned);
extern const struct file_operations btrfs_file_operations;
int __btrfs_drop_extents(struct btrfs_trans_handle *trans,
struct btrfs_root *root, struct inode *inode, u64 start,
u64 end, int drop_cache);
int btrfs_mark_extent_written(struct btrfs_trans_handle *trans,
- struct inode *inode, u64 start, u64 end);
+ struct btrfs_inode *inode, u64 start, u64 end);
int btrfs_release_file(struct inode *inode, struct file *file);
int btrfs_dirty_pages(struct inode *inode, struct page **pages,
size_t num_pages, loff_t pos, size_t write_bytes,
i_uid_write(inode, btrfs_stack_inode_uid(inode_item));
i_gid_write(inode, btrfs_stack_inode_gid(inode_item));
- btrfs_i_size_write(inode, btrfs_stack_inode_size(inode_item));
+ btrfs_i_size_write(BTRFS_I(inode), btrfs_stack_inode_size(inode_item));
inode->i_mode = btrfs_stack_inode_mode(inode_item);
set_nlink(inode, btrfs_stack_inode_nlink(inode_item));
inode_set_bytes(inode, btrfs_stack_inode_nbytes(inode_item));
dev_replace->cursor_left_last_write_of_item;
}
-int btrfs_dev_replace_start(struct btrfs_fs_info *fs_info, char *tgtdev_name,
- u64 srcdevid, char *srcdev_name, int read_src)
+int btrfs_dev_replace_start(struct btrfs_fs_info *fs_info,
+ const char *tgtdev_name, u64 srcdevid, const char *srcdev_name,
+ int read_src)
{
struct btrfs_root *root = fs_info->dev_root;
struct btrfs_trans_handle *trans;
void btrfs_after_dev_replace_commit(struct btrfs_fs_info *fs_info);
int btrfs_dev_replace_by_ioctl(struct btrfs_fs_info *fs_info,
struct btrfs_ioctl_dev_replace_args *args);
-int btrfs_dev_replace_start(struct btrfs_fs_info *fs_info, char *tgtdev_name,
- u64 srcdevid, char *srcdev_name, int read_src);
+int btrfs_dev_replace_start(struct btrfs_fs_info *fs_info,
+ const char *tgtdev_name, u64 srcdevid, const char *srcdev_name,
+ int read_src);
void btrfs_dev_replace_status(struct btrfs_fs_info *fs_info,
struct btrfs_ioctl_dev_replace_args *args);
int btrfs_dev_replace_cancel(struct btrfs_fs_info *fs_info,
struct extent_buffer *leaf;
u32 data_size;
- BUG_ON(name_len + data_len > BTRFS_MAX_XATTR_SIZE(root->fs_info));
+ if (name_len + data_len > BTRFS_MAX_XATTR_SIZE(root->fs_info))
+ return -ENOSPC;
key.objectid = objectid;
key.type = BTRFS_XATTR_ITEM_KEY;
*/
int btrfs_insert_dir_item(struct btrfs_trans_handle *trans, struct btrfs_root
*root, const char *name, int name_len,
- struct inode *dir, struct btrfs_key *location,
+ struct btrfs_inode *dir, struct btrfs_key *location,
u8 type, u64 index)
{
int ret = 0;
struct btrfs_disk_key disk_key;
u32 data_size;
- key.objectid = btrfs_ino(BTRFS_I(dir));
+ key.objectid = btrfs_ino(dir);
key.type = BTRFS_DIR_ITEM_KEY;
key.offset = btrfs_name_hash(name, name_len);
btrfs_release_path(path);
ret2 = btrfs_insert_delayed_dir_index(trans, root->fs_info, name,
- name_len, BTRFS_I(dir), &disk_key, type, index);
+ name_len, dir, &disk_key, type, index);
out_free:
btrfs_free_path(path);
if (ret)
* extents on the btree inode are pretty simple, there's one extent
* that covers the entire device
*/
-static struct extent_map *btree_get_extent(struct inode *inode,
+static struct extent_map *btree_get_extent(struct btrfs_inode *inode,
struct page *page, size_t pg_offset, u64 start, u64 len,
int create)
{
- struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
- struct extent_map_tree *em_tree = &BTRFS_I(inode)->extent_tree;
+ struct btrfs_fs_info *fs_info = btrfs_sb(inode->vfs_inode.i_sb);
+ struct extent_map_tree *em_tree = &inode->extent_tree;
struct extent_map *em;
int ret;
return em;
}
-u32 btrfs_csum_data(char *data, u32 seed, size_t len)
+u32 btrfs_csum_data(const char *data, u32 seed, size_t len)
{
return btrfs_crc32c(seed, data, len);
}
btrfs_destroy_workqueue(fs_info->delalloc_workers);
btrfs_destroy_workqueue(fs_info->workers);
btrfs_destroy_workqueue(fs_info->endio_workers);
- btrfs_destroy_workqueue(fs_info->endio_meta_workers);
btrfs_destroy_workqueue(fs_info->endio_raid56_workers);
btrfs_destroy_workqueue(fs_info->endio_repair_workers);
btrfs_destroy_workqueue(fs_info->rmw_workers);
- btrfs_destroy_workqueue(fs_info->endio_meta_write_workers);
btrfs_destroy_workqueue(fs_info->endio_write_workers);
btrfs_destroy_workqueue(fs_info->endio_freespace_worker);
btrfs_destroy_workqueue(fs_info->submit_workers);
btrfs_destroy_workqueue(fs_info->flush_workers);
btrfs_destroy_workqueue(fs_info->qgroup_rescan_workers);
btrfs_destroy_workqueue(fs_info->extent_workers);
+ /*
+ * Now that all other work queues are destroyed, we can safely destroy
+ * the queues used for metadata I/O, since tasks from those other work
+ * queues can do metadata I/O operations.
+ */
+ btrfs_destroy_workqueue(fs_info->endio_meta_workers);
+ btrfs_destroy_workqueue(fs_info->endio_meta_write_workers);
}
static void free_root_extent_buffers(struct btrfs_root *root)
fail_block_groups:
btrfs_put_block_group_cache(fs_info);
- btrfs_free_block_groups(fs_info);
fail_tree_roots:
free_root_pointers(fs_info, 1);
fail_sb_buffer:
btrfs_stop_all_workers(fs_info);
+ btrfs_free_block_groups(fs_info);
fail_alloc:
fail_iput:
btrfs_mapping_tree_free(&fs_info->mapping_tree);
btrfs_set_super_bytenr(sb, bytenr);
crc = ~(u32)0;
- crc = btrfs_csum_data((char *)sb +
+ crc = btrfs_csum_data((const char *)sb +
BTRFS_CSUM_SIZE, crc,
BTRFS_SUPER_INFO_SIZE -
BTRFS_CSUM_SIZE);
btrfs_put_block_group_cache(fs_info);
- btrfs_free_block_groups(fs_info);
-
/*
* we must make sure there is not any read request to
* submit after we stopping all workers.
invalidate_inode_pages2(fs_info->btree_inode->i_mapping);
btrfs_stop_all_workers(fs_info);
+ btrfs_free_block_groups(fs_info);
+
clear_bit(BTRFS_FS_OPEN, &fs_info->flags);
free_root_pointers(fs_info, 1);
}
static const struct extent_io_ops btree_extent_io_ops = {
- .readpage_end_io_hook = btree_readpage_end_io_hook,
- .readpage_io_failed_hook = btree_io_failed_hook,
+ /* mandatory callbacks */
.submit_bio_hook = btree_submit_bio_hook,
+ .readpage_end_io_hook = btree_readpage_end_io_hook,
/* note we're sharing with inode.c for the merge bio hook */
.merge_bio_hook = btrfs_merge_bio_hook,
+ .readpage_io_failed_hook = btree_io_failed_hook,
+
+ /* optional callbacks */
};
int btrfs_buffer_uptodate(struct extent_buffer *buf, u64 parent_transid,
int atomic);
int btrfs_read_buffer(struct extent_buffer *buf, u64 parent_transid);
-u32 btrfs_csum_data(char *data, u32 seed, size_t len);
+u32 btrfs_csum_data(const char *data, u32 seed, size_t len);
void btrfs_csum_final(u32 crc, u8 *result);
int btrfs_bio_wq_end_io(struct btrfs_fs_info *info, struct bio *bio,
enum btrfs_wq_endio_type metadata);
* Boston, MA 021110-1307, USA.
*/
#include <linux/sched.h>
+#include <linux/sched/signal.h>
#include <linux/pagemap.h>
#include <linux/writeback.h>
#include <linux/blkdev.h>
(may_use_included ? s_info->bytes_may_use : 0);
}
-int btrfs_alloc_data_chunk_ondemand(struct inode *inode, u64 bytes)
+int btrfs_alloc_data_chunk_ondemand(struct btrfs_inode *inode, u64 bytes)
{
struct btrfs_space_info *data_sinfo;
- struct btrfs_root *root = BTRFS_I(inode)->root;
+ struct btrfs_root *root = inode->root;
struct btrfs_fs_info *fs_info = root->fs_info;
u64 used;
int ret = 0;
round_down(start, fs_info->sectorsize);
start = round_down(start, fs_info->sectorsize);
- ret = btrfs_alloc_data_chunk_ondemand(inode, len);
+ ret = btrfs_alloc_data_chunk_ondemand(BTRFS_I(inode), len);
if (ret < 0)
return ret;
/* Can only return 0 or -ENOSPC */
int btrfs_orphan_reserve_metadata(struct btrfs_trans_handle *trans,
- struct inode *inode)
+ struct btrfs_inode *inode)
{
- struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
- struct btrfs_root *root = BTRFS_I(inode)->root;
+ struct btrfs_fs_info *fs_info = btrfs_sb(inode->vfs_inode.i_sb);
+ struct btrfs_root *root = inode->root;
/*
* We always use trans->block_rsv here as we will have reserved space
* for our orphan when starting the transaction, using get_block_rsv()
*/
u64 num_bytes = btrfs_calc_trans_metadata_size(fs_info, 1);
- trace_btrfs_space_reservation(fs_info, "orphan",
- btrfs_ino(BTRFS_I(inode)), num_bytes, 1);
+ trace_btrfs_space_reservation(fs_info, "orphan", btrfs_ino(inode),
+ num_bytes, 1);
return btrfs_block_rsv_migrate(src_rsv, dst_rsv, num_bytes, 1);
}
-void btrfs_orphan_release_metadata(struct inode *inode)
+void btrfs_orphan_release_metadata(struct btrfs_inode *inode)
{
- struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
- struct btrfs_root *root = BTRFS_I(inode)->root;
+ struct btrfs_fs_info *fs_info = btrfs_sb(inode->vfs_inode.i_sb);
+ struct btrfs_root *root = inode->root;
u64 num_bytes = btrfs_calc_trans_metadata_size(fs_info, 1);
- trace_btrfs_space_reservation(fs_info, "orphan",
- btrfs_ino(BTRFS_I(inode)), num_bytes, 0);
+ trace_btrfs_space_reservation(fs_info, "orphan", btrfs_ino(inode),
+ num_bytes, 0);
btrfs_block_rsv_release(fs_info, root->orphan_block_rsv, num_bytes);
}
* reserved extents that need to be freed. This must be called with
* BTRFS_I(inode)->lock held.
*/
-static unsigned drop_outstanding_extent(struct inode *inode, u64 num_bytes)
+static unsigned drop_outstanding_extent(struct btrfs_inode *inode,
+ u64 num_bytes)
{
unsigned drop_inode_space = 0;
unsigned dropped_extents = 0;
num_extents = count_max_extents(num_bytes);
ASSERT(num_extents);
- ASSERT(BTRFS_I(inode)->outstanding_extents >= num_extents);
- BTRFS_I(inode)->outstanding_extents -= num_extents;
+ ASSERT(inode->outstanding_extents >= num_extents);
+ inode->outstanding_extents -= num_extents;
- if (BTRFS_I(inode)->outstanding_extents == 0 &&
+ if (inode->outstanding_extents == 0 &&
test_and_clear_bit(BTRFS_INODE_DELALLOC_META_RESERVED,
- &BTRFS_I(inode)->runtime_flags))
+ &inode->runtime_flags))
drop_inode_space = 1;
/*
* If we have more or the same amount of outstanding extents than we have
* reserved then we need to leave the reserved extents count alone.
*/
- if (BTRFS_I(inode)->outstanding_extents >=
- BTRFS_I(inode)->reserved_extents)
+ if (inode->outstanding_extents >= inode->reserved_extents)
return drop_inode_space;
- dropped_extents = BTRFS_I(inode)->reserved_extents -
- BTRFS_I(inode)->outstanding_extents;
- BTRFS_I(inode)->reserved_extents -= dropped_extents;
+ dropped_extents = inode->reserved_extents - inode->outstanding_extents;
+ inode->reserved_extents -= dropped_extents;
return dropped_extents + drop_inode_space;
}
*
* This must be called with BTRFS_I(inode)->lock held.
*/
-static u64 calc_csum_metadata_size(struct inode *inode, u64 num_bytes,
+static u64 calc_csum_metadata_size(struct btrfs_inode *inode, u64 num_bytes,
int reserve)
{
- struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
+ struct btrfs_fs_info *fs_info = btrfs_sb(inode->vfs_inode.i_sb);
u64 old_csums, num_csums;
- if (BTRFS_I(inode)->flags & BTRFS_INODE_NODATASUM &&
- BTRFS_I(inode)->csum_bytes == 0)
+ if (inode->flags & BTRFS_INODE_NODATASUM && inode->csum_bytes == 0)
return 0;
- old_csums = btrfs_csum_bytes_to_leaves(fs_info,
- BTRFS_I(inode)->csum_bytes);
+ old_csums = btrfs_csum_bytes_to_leaves(fs_info, inode->csum_bytes);
if (reserve)
- BTRFS_I(inode)->csum_bytes += num_bytes;
+ inode->csum_bytes += num_bytes;
else
- BTRFS_I(inode)->csum_bytes -= num_bytes;
- num_csums = btrfs_csum_bytes_to_leaves(fs_info,
- BTRFS_I(inode)->csum_bytes);
+ inode->csum_bytes -= num_bytes;
+ num_csums = btrfs_csum_bytes_to_leaves(fs_info, inode->csum_bytes);
/* No change, no need to reserve more */
if (old_csums == num_csums)
return btrfs_calc_trans_metadata_size(fs_info, old_csums - num_csums);
}
-int btrfs_delalloc_reserve_metadata(struct inode *inode, u64 num_bytes)
+int btrfs_delalloc_reserve_metadata(struct btrfs_inode *inode, u64 num_bytes)
{
- struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
- struct btrfs_root *root = BTRFS_I(inode)->root;
+ struct btrfs_fs_info *fs_info = btrfs_sb(inode->vfs_inode.i_sb);
+ struct btrfs_root *root = inode->root;
struct btrfs_block_rsv *block_rsv = &fs_info->delalloc_block_rsv;
u64 to_reserve = 0;
u64 csum_bytes;
schedule_timeout(1);
if (delalloc_lock)
- mutex_lock(&BTRFS_I(inode)->delalloc_mutex);
+ mutex_lock(&inode->delalloc_mutex);
num_bytes = ALIGN(num_bytes, fs_info->sectorsize);
- spin_lock(&BTRFS_I(inode)->lock);
+ spin_lock(&inode->lock);
nr_extents = count_max_extents(num_bytes);
- BTRFS_I(inode)->outstanding_extents += nr_extents;
+ inode->outstanding_extents += nr_extents;
nr_extents = 0;
- if (BTRFS_I(inode)->outstanding_extents >
- BTRFS_I(inode)->reserved_extents)
- nr_extents += BTRFS_I(inode)->outstanding_extents -
- BTRFS_I(inode)->reserved_extents;
+ if (inode->outstanding_extents > inode->reserved_extents)
+ nr_extents += inode->outstanding_extents -
+ inode->reserved_extents;
/* We always want to reserve a slot for updating the inode. */
to_reserve = btrfs_calc_trans_metadata_size(fs_info, nr_extents + 1);
to_reserve += calc_csum_metadata_size(inode, num_bytes, 1);
- csum_bytes = BTRFS_I(inode)->csum_bytes;
- spin_unlock(&BTRFS_I(inode)->lock);
+ csum_bytes = inode->csum_bytes;
+ spin_unlock(&inode->lock);
if (test_bit(BTRFS_FS_QUOTA_ENABLED, &fs_info->flags)) {
ret = btrfs_qgroup_reserve_meta(root,
goto out_fail;
}
- spin_lock(&BTRFS_I(inode)->lock);
+ spin_lock(&inode->lock);
if (test_and_set_bit(BTRFS_INODE_DELALLOC_META_RESERVED,
- &BTRFS_I(inode)->runtime_flags)) {
+ &inode->runtime_flags)) {
to_reserve -= btrfs_calc_trans_metadata_size(fs_info, 1);
release_extra = true;
}
- BTRFS_I(inode)->reserved_extents += nr_extents;
- spin_unlock(&BTRFS_I(inode)->lock);
+ inode->reserved_extents += nr_extents;
+ spin_unlock(&inode->lock);
if (delalloc_lock)
- mutex_unlock(&BTRFS_I(inode)->delalloc_mutex);
+ mutex_unlock(&inode->delalloc_mutex);
if (to_reserve)
trace_btrfs_space_reservation(fs_info, "delalloc",
- btrfs_ino(BTRFS_I(inode)), to_reserve, 1);
+ btrfs_ino(inode), to_reserve, 1);
if (release_extra)
btrfs_block_rsv_release(fs_info, block_rsv,
btrfs_calc_trans_metadata_size(fs_info, 1));
return 0;
out_fail:
- spin_lock(&BTRFS_I(inode)->lock);
+ spin_lock(&inode->lock);
dropped = drop_outstanding_extent(inode, num_bytes);
/*
* If the inodes csum_bytes is the same as the original
* csum_bytes then we know we haven't raced with any free()ers
* so we can just reduce our inodes csum bytes and carry on.
*/
- if (BTRFS_I(inode)->csum_bytes == csum_bytes) {
+ if (inode->csum_bytes == csum_bytes) {
calc_csum_metadata_size(inode, num_bytes, 0);
} else {
- u64 orig_csum_bytes = BTRFS_I(inode)->csum_bytes;
+ u64 orig_csum_bytes = inode->csum_bytes;
u64 bytes;
/*
* number of bytes that were freed while we were trying our
* reservation.
*/
- bytes = csum_bytes - BTRFS_I(inode)->csum_bytes;
- BTRFS_I(inode)->csum_bytes = csum_bytes;
+ bytes = csum_bytes - inode->csum_bytes;
+ inode->csum_bytes = csum_bytes;
to_free = calc_csum_metadata_size(inode, bytes, 0);
* been making this reservation and our ->csum_bytes were not
* artificially inflated.
*/
- BTRFS_I(inode)->csum_bytes = csum_bytes - num_bytes;
+ inode->csum_bytes = csum_bytes - num_bytes;
bytes = csum_bytes - orig_csum_bytes;
bytes = calc_csum_metadata_size(inode, bytes, 0);
* need to do anything, the other free-ers did the correct
* thing.
*/
- BTRFS_I(inode)->csum_bytes = orig_csum_bytes - num_bytes;
+ inode->csum_bytes = orig_csum_bytes - num_bytes;
if (bytes > to_free)
to_free = bytes - to_free;
else
to_free = 0;
}
- spin_unlock(&BTRFS_I(inode)->lock);
+ spin_unlock(&inode->lock);
if (dropped)
to_free += btrfs_calc_trans_metadata_size(fs_info, dropped);
if (to_free) {
btrfs_block_rsv_release(fs_info, block_rsv, to_free);
trace_btrfs_space_reservation(fs_info, "delalloc",
- btrfs_ino(BTRFS_I(inode)), to_free, 0);
+ btrfs_ino(inode), to_free, 0);
}
if (delalloc_lock)
- mutex_unlock(&BTRFS_I(inode)->delalloc_mutex);
+ mutex_unlock(&inode->delalloc_mutex);
return ret;
}
* once we complete IO for a given set of bytes to release their metadata
* reservations.
*/
-void btrfs_delalloc_release_metadata(struct inode *inode, u64 num_bytes)
+void btrfs_delalloc_release_metadata(struct btrfs_inode *inode, u64 num_bytes)
{
- struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
+ struct btrfs_fs_info *fs_info = btrfs_sb(inode->vfs_inode.i_sb);
u64 to_free = 0;
unsigned dropped;
num_bytes = ALIGN(num_bytes, fs_info->sectorsize);
- spin_lock(&BTRFS_I(inode)->lock);
+ spin_lock(&inode->lock);
dropped = drop_outstanding_extent(inode, num_bytes);
if (num_bytes)
to_free = calc_csum_metadata_size(inode, num_bytes, 0);
- spin_unlock(&BTRFS_I(inode)->lock);
+ spin_unlock(&inode->lock);
if (dropped > 0)
to_free += btrfs_calc_trans_metadata_size(fs_info, dropped);
if (btrfs_is_testing(fs_info))
return;
- trace_btrfs_space_reservation(fs_info, "delalloc",
- btrfs_ino(BTRFS_I(inode)), to_free, 0);
+ trace_btrfs_space_reservation(fs_info, "delalloc", btrfs_ino(inode),
+ to_free, 0);
btrfs_block_rsv_release(fs_info, &fs_info->delalloc_block_rsv, to_free);
}
ret = btrfs_check_data_free_space(inode, start, len);
if (ret < 0)
return ret;
- ret = btrfs_delalloc_reserve_metadata(inode, len);
+ ret = btrfs_delalloc_reserve_metadata(BTRFS_I(inode), len);
if (ret < 0)
btrfs_free_reserved_data_space(inode, start, len);
return ret;
*/
void btrfs_delalloc_release_space(struct inode *inode, u64 start, u64 len)
{
- btrfs_delalloc_release_metadata(inode, len);
+ btrfs_delalloc_release_metadata(BTRFS_I(inode), len);
btrfs_free_reserved_data_space(inode, start, len);
}
}
}
+/*
+ * Must be called only after stopping all workers, since we could have block
+ * group caching kthreads running, and therefore they could race with us if we
+ * freed the block groups before stopping them.
+ */
int btrfs_free_block_groups(struct btrfs_fs_info *info)
{
struct btrfs_block_group_cache *block_group;
list_del(&block_group->list);
up_write(&block_group->space_info->groups_sem);
- if (block_group->cached == BTRFS_CACHE_STARTED)
- wait_block_group_cache_done(block_group);
-
/*
* We haven't cached this block group, which means we could
* possibly have excluded extents on this block group.
free_excluded_extents(info, block_group);
btrfs_remove_free_space_cache(block_group);
+ ASSERT(block_group->cached != BTRFS_CACHE_STARTED);
ASSERT(list_empty(&block_group->dirty_list));
ASSERT(list_empty(&block_group->io_list));
ASSERT(list_empty(&block_group->bg_list));
mutex_unlock(&trans->transaction->cache_write_mutex);
if (!IS_ERR(inode)) {
- ret = btrfs_orphan_add(trans, inode);
+ ret = btrfs_orphan_add(trans, BTRFS_I(inode));
if (ret) {
btrfs_add_delayed_iput(inode);
goto out;
struct extent_state *state, unsigned *bits)
{
if (tree->ops && tree->ops->clear_bit_hook)
- tree->ops->clear_bit_hook(tree->mapping->host, state, bits);
+ tree->ops->clear_bit_hook(BTRFS_I(tree->mapping->host),
+ state, bits);
}
static void set_state_bits(struct extent_io_tree *tree,
SetPageUptodate(page);
}
-int free_io_failure(struct inode *inode, struct io_failure_record *rec)
+int free_io_failure(struct btrfs_inode *inode, struct io_failure_record *rec)
{
int ret;
int err = 0;
- struct extent_io_tree *failure_tree = &BTRFS_I(inode)->io_failure_tree;
+ struct extent_io_tree *failure_tree = &inode->io_failure_tree;
set_state_failrec(failure_tree, rec->start, NULL);
ret = clear_extent_bits(failure_tree, rec->start,
if (ret)
err = ret;
- ret = clear_extent_bits(&BTRFS_I(inode)->io_tree, rec->start,
+ ret = clear_extent_bits(&inode->io_tree, rec->start,
rec->start + rec->len - 1,
EXTENT_DAMAGED);
if (ret && !err)
* currently, there can be no more than two copies of every data bit. thus,
* exactly one rewrite is required.
*/
-int repair_io_failure(struct inode *inode, u64 start, u64 length, u64 logical,
- struct page *page, unsigned int pg_offset, int mirror_num)
+int repair_io_failure(struct btrfs_inode *inode, u64 start, u64 length,
+ u64 logical, struct page *page,
+ unsigned int pg_offset, int mirror_num)
{
- struct btrfs_fs_info *fs_info = BTRFS_I(inode)->root->fs_info;
+ struct btrfs_fs_info *fs_info = inode->root->fs_info;
struct bio *bio;
struct btrfs_device *dev;
u64 map_length = 0;
btrfs_info_rl_in_rcu(fs_info,
"read error corrected: ino %llu off %llu (dev %s sector %llu)",
- btrfs_ino(BTRFS_I(inode)), start,
+ btrfs_ino(inode), start,
rcu_str_deref(dev->name), sector);
btrfs_bio_counter_dec(fs_info);
bio_put(bio);
for (i = 0; i < num_pages; i++) {
struct page *p = eb->pages[i];
- ret = repair_io_failure(fs_info->btree_inode, start,
+ ret = repair_io_failure(BTRFS_I(fs_info->btree_inode), start,
PAGE_SIZE, start, p,
start - page_offset(p), mirror_num);
if (ret)
* each time an IO finishes, we do a fast check in the IO failure tree
* to see if we need to process or clean up an io_failure_record
*/
-int clean_io_failure(struct inode *inode, u64 start, struct page *page,
+int clean_io_failure(struct btrfs_inode *inode, u64 start, struct page *page,
unsigned int pg_offset)
{
u64 private;
struct io_failure_record *failrec;
- struct btrfs_fs_info *fs_info = BTRFS_I(inode)->root->fs_info;
+ struct btrfs_fs_info *fs_info = inode->root->fs_info;
struct extent_state *state;
int num_copies;
int ret;
private = 0;
- ret = count_range_bits(&BTRFS_I(inode)->io_failure_tree, &private,
+ ret = count_range_bits(&inode->io_failure_tree, &private,
(u64)-1, 1, EXTENT_DIRTY, 0);
if (!ret)
return 0;
- ret = get_state_failrec(&BTRFS_I(inode)->io_failure_tree, start,
+ ret = get_state_failrec(&inode->io_failure_tree, start,
&failrec);
if (ret)
return 0;
if (fs_info->sb->s_flags & MS_RDONLY)
goto out;
- spin_lock(&BTRFS_I(inode)->io_tree.lock);
- state = find_first_extent_bit_state(&BTRFS_I(inode)->io_tree,
+ spin_lock(&inode->io_tree.lock);
+ state = find_first_extent_bit_state(&inode->io_tree,
failrec->start,
EXTENT_LOCKED);
- spin_unlock(&BTRFS_I(inode)->io_tree.lock);
+ spin_unlock(&inode->io_tree.lock);
if (state && state->start <= failrec->start &&
state->end >= failrec->start + failrec->len - 1) {
* - under ordered extent
* - the inode is freeing
*/
-void btrfs_free_io_failure_record(struct inode *inode, u64 start, u64 end)
+void btrfs_free_io_failure_record(struct btrfs_inode *inode, u64 start, u64 end)
{
- struct extent_io_tree *failure_tree = &BTRFS_I(inode)->io_failure_tree;
+ struct extent_io_tree *failure_tree = &inode->io_failure_tree;
struct io_failure_record *failrec;
struct extent_state *state, *next;
ret = btrfs_check_repairable(inode, failed_bio, failrec, failed_mirror);
if (!ret) {
- free_io_failure(inode, failrec);
+ free_io_failure(BTRFS_I(inode), failrec);
return -EIO;
}
(int)phy_offset, failed_bio->bi_end_io,
NULL);
if (!bio) {
- free_io_failure(inode, failrec);
+ free_io_failure(BTRFS_I(inode), failrec);
return -EIO;
}
bio_set_op_attrs(bio, REQ_OP_READ, read_mode);
ret = tree->ops->submit_bio_hook(inode, bio, failrec->this_mirror,
failrec->bio_flags, 0);
if (ret) {
- free_io_failure(inode, failrec);
+ free_io_failure(BTRFS_I(inode), failrec);
bio_put(bio);
}
tree = &BTRFS_I(page->mapping->host)->io_tree;
- if (tree->ops && tree->ops->writepage_end_io_hook) {
- ret = tree->ops->writepage_end_io_hook(page, start,
- end, NULL, uptodate);
- if (ret)
- uptodate = 0;
- }
+ if (tree->ops && tree->ops->writepage_end_io_hook)
+ tree->ops->writepage_end_io_hook(page, start, end, NULL,
+ uptodate);
if (!uptodate) {
ClearPageUptodate(page);
len = bvec->bv_len;
mirror = io_bio->mirror_num;
- if (likely(uptodate && tree->ops &&
- tree->ops->readpage_end_io_hook)) {
+ if (likely(uptodate && tree->ops)) {
ret = tree->ops->readpage_end_io_hook(io_bio, offset,
page, start, end,
mirror);
if (ret)
uptodate = 0;
else
- clean_io_failure(inode, start, page, 0);
+ clean_io_failure(BTRFS_I(inode), start,
+ page, 0);
}
if (likely(uptodate))
goto readpage_ok;
- if (tree->ops && tree->ops->readpage_io_failed_hook) {
+ if (tree->ops) {
ret = tree->ops->readpage_io_failed_hook(page, mirror);
if (!ret && !bio->bi_error)
uptodate = 1;
bio->bi_private = NULL;
bio_get(bio);
- if (tree->ops && tree->ops->submit_bio_hook)
+ if (tree->ops)
ret = tree->ops->submit_bio_hook(page->mapping->host, bio,
mirror_num, bio_flags, start);
else
unsigned long bio_flags)
{
int ret = 0;
- if (tree->ops && tree->ops->merge_bio_hook)
+ if (tree->ops)
ret = tree->ops->merge_bio_hook(page, offset, size, bio,
bio_flags);
return ret;
*em_cached = NULL;
}
- em = get_extent(inode, page, pg_offset, start, len, 0);
+ em = get_extent(BTRFS_I(inode), page, pg_offset, start, len, 0);
if (em_cached && !IS_ERR_OR_NULL(em)) {
BUG_ON(*em_cached);
atomic_inc(&em->refs);
inode = pages[0]->mapping->host;
while (1) {
lock_extent(tree, start, end);
- ordered = btrfs_lookup_ordered_range(inode, start,
+ ordered = btrfs_lookup_ordered_range(BTRFS_I(inode), start,
end - start + 1);
if (!ordered)
break;
while (1) {
lock_extent(tree, start, end);
- ordered = btrfs_lookup_ordered_range(inode, start,
+ ordered = btrfs_lookup_ordered_range(BTRFS_I(inode), start,
PAGE_SIZE);
if (!ordered)
break;
page_end, NULL, 1);
break;
}
- em = epd->get_extent(inode, page, pg_offset, cur,
+ em = epd->get_extent(BTRFS_I(inode), page, pg_offset, cur,
end - cur + 1, 1);
if (IS_ERR_OR_NULL(em)) {
SetPageError(page);
if (len == 0)
break;
len = ALIGN(len, sectorsize);
- em = get_extent(inode, NULL, 0, offset, len, 0);
+ em = get_extent(BTRFS_I(inode), NULL, 0, offset, len, 0);
if (IS_ERR_OR_NULL(em))
return em;
struct extent_state;
struct btrfs_root;
+struct btrfs_inode;
struct btrfs_io_bio;
struct io_failure_record;
int mirror_num, unsigned long bio_flags,
u64 bio_offset);
struct extent_io_ops {
- int (*fill_delalloc)(struct inode *inode, struct page *locked_page,
- u64 start, u64 end, int *page_started,
- unsigned long *nr_written);
- int (*writepage_start_hook)(struct page *page, u64 start, u64 end);
+ /*
+ * The following callbacks must be allways defined, the function
+ * pointer will be called unconditionally.
+ */
extent_submit_bio_hook_t *submit_bio_hook;
+ int (*readpage_end_io_hook)(struct btrfs_io_bio *io_bio, u64 phy_offset,
+ struct page *page, u64 start, u64 end,
+ int mirror);
int (*merge_bio_hook)(struct page *page, unsigned long offset,
size_t size, struct bio *bio,
unsigned long bio_flags);
int (*readpage_io_failed_hook)(struct page *page, int failed_mirror);
- int (*readpage_end_io_hook)(struct btrfs_io_bio *io_bio, u64 phy_offset,
- struct page *page, u64 start, u64 end,
- int mirror);
- int (*writepage_end_io_hook)(struct page *page, u64 start, u64 end,
+
+ /*
+ * Optional hooks, called if the pointer is not NULL
+ */
+ int (*fill_delalloc)(struct inode *inode, struct page *locked_page,
+ u64 start, u64 end, int *page_started,
+ unsigned long *nr_written);
+
+ int (*writepage_start_hook)(struct page *page, u64 start, u64 end);
+ void (*writepage_end_io_hook)(struct page *page, u64 start, u64 end,
struct extent_state *state, int uptodate);
void (*set_bit_hook)(struct inode *inode, struct extent_state *state,
unsigned *bits);
- void (*clear_bit_hook)(struct inode *inode, struct extent_state *state,
- unsigned *bits);
+ void (*clear_bit_hook)(struct btrfs_inode *inode,
+ struct extent_state *state,
+ unsigned *bits);
void (*merge_extent_hook)(struct inode *inode,
struct extent_state *new,
struct extent_state *other);
struct extent_map_tree;
-typedef struct extent_map *(get_extent_t)(struct inode *inode,
+typedef struct extent_map *(get_extent_t)(struct btrfs_inode *inode,
struct page *page,
size_t pg_offset,
u64 start, u64 len,
struct bio *btrfs_bio_clone(struct bio *bio, gfp_t gfp_mask);
struct btrfs_fs_info;
+struct btrfs_inode;
-int repair_io_failure(struct inode *inode, u64 start, u64 length, u64 logical,
- struct page *page, unsigned int pg_offset,
- int mirror_num);
-int clean_io_failure(struct inode *inode, u64 start, struct page *page,
- unsigned int pg_offset);
+int repair_io_failure(struct btrfs_inode *inode, u64 start, u64 length,
+ u64 logical, struct page *page,
+ unsigned int pg_offset, int mirror_num);
+int clean_io_failure(struct btrfs_inode *inode, u64 start,
+ struct page *page, unsigned int pg_offset);
void end_extent_writepage(struct page *page, int err, u64 start, u64 end);
int repair_eb_io_failure(struct btrfs_fs_info *fs_info,
struct extent_buffer *eb, int mirror_num);
int in_validation;
};
-void btrfs_free_io_failure_record(struct inode *inode, u64 start, u64 end);
+
+void btrfs_free_io_failure_record(struct btrfs_inode *inode, u64 start,
+ u64 end);
int btrfs_get_io_failure_record(struct inode *inode, u64 start, u64 end,
struct io_failure_record **failrec_ret);
int btrfs_check_repairable(struct inode *inode, struct bio *failed_bio,
struct io_failure_record *failrec,
struct page *page, int pg_offset, int icsum,
bio_end_io_t *endio_func, void *data);
-int free_io_failure(struct inode *inode, struct io_failure_record *rec);
+int free_io_failure(struct btrfs_inode *inode, struct io_failure_record *rec);
#ifdef CONFIG_BTRFS_FS_RUN_SANITY_TESTS
noinline u64 find_lock_delalloc_range(struct inode *inode,
struct extent_io_tree *tree,
* read from the commit root and sidestep a nasty deadlock
* between reading the free space cache and updating the csum tree.
*/
- if (btrfs_is_free_space_inode(inode)) {
+ if (btrfs_is_free_space_inode(BTRFS_I(inode))) {
path->search_commit_root = 1;
path->skip_locking = 1;
}
/* delete the entire item, it is inside our range */
if (key.offset >= bytenr && csum_end <= end_byte) {
- ret = btrfs_del_item(trans, root, path);
+ int del_nr = 1;
+
+ /*
+ * Check how many csum items preceding this one in this
+ * leaf correspond to our range and then delete them all
+ * at once.
+ */
+ if (key.offset > bytenr && path->slots[0] > 0) {
+ int slot = path->slots[0] - 1;
+
+ while (slot >= 0) {
+ struct btrfs_key pk;
+
+ btrfs_item_key_to_cpu(leaf, &pk, slot);
+ if (pk.offset < bytenr ||
+ pk.type != BTRFS_EXTENT_CSUM_KEY ||
+ pk.objectid !=
+ BTRFS_EXTENT_CSUM_OBJECTID)
+ break;
+ path->slots[0] = slot;
+ del_nr++;
+ key.offset = pk.offset;
+ slot--;
+ }
+ }
+ ret = btrfs_del_items(trans, root, path,
+ path->slots[0], del_nr);
if (ret)
goto out;
if (key.offset == bytenr)
goto out;
}
-void btrfs_extent_item_to_extent_map(struct inode *inode,
+void btrfs_extent_item_to_extent_map(struct btrfs_inode *inode,
const struct btrfs_path *path,
struct btrfs_file_extent_item *fi,
const bool new_inline,
struct extent_map *em)
{
- struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
- struct btrfs_root *root = BTRFS_I(inode)->root;
+ struct btrfs_fs_info *fs_info = btrfs_sb(inode->vfs_inode.i_sb);
+ struct btrfs_root *root = inode->root;
struct extent_buffer *leaf = path->nodes[0];
const int slot = path->slots[0];
struct btrfs_key key;
}
} else {
btrfs_err(fs_info,
- "unknown file extent item type %d, inode %llu, offset %llu, root %llu",
- type, btrfs_ino(BTRFS_I(inode)), extent_start,
+ "unknown file extent item type %d, inode %llu, offset %llu, "
+ "root %llu", type, btrfs_ino(inode), extent_start,
root->root_key.objectid);
}
}
* If an existing record is found the defrag item you
* pass in is freed
*/
-static int __btrfs_add_inode_defrag(struct inode *inode,
+static int __btrfs_add_inode_defrag(struct btrfs_inode *inode,
struct inode_defrag *defrag)
{
- struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
+ struct btrfs_fs_info *fs_info = btrfs_sb(inode->vfs_inode.i_sb);
struct inode_defrag *entry;
struct rb_node **p;
struct rb_node *parent = NULL;
return -EEXIST;
}
}
- set_bit(BTRFS_INODE_IN_DEFRAG, &BTRFS_I(inode)->runtime_flags);
+ set_bit(BTRFS_INODE_IN_DEFRAG, &inode->runtime_flags);
rb_link_node(&defrag->rb_node, parent, p);
rb_insert_color(&defrag->rb_node, &fs_info->defrag_inodes);
return 0;
* enabled
*/
int btrfs_add_inode_defrag(struct btrfs_trans_handle *trans,
- struct inode *inode)
+ struct btrfs_inode *inode)
{
- struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
- struct btrfs_root *root = BTRFS_I(inode)->root;
+ struct btrfs_fs_info *fs_info = btrfs_sb(inode->vfs_inode.i_sb);
+ struct btrfs_root *root = inode->root;
struct inode_defrag *defrag;
u64 transid;
int ret;
if (!__need_auto_defrag(fs_info))
return 0;
- if (test_bit(BTRFS_INODE_IN_DEFRAG, &BTRFS_I(inode)->runtime_flags))
+ if (test_bit(BTRFS_INODE_IN_DEFRAG, &inode->runtime_flags))
return 0;
if (trans)
transid = trans->transid;
else
- transid = BTRFS_I(inode)->root->last_trans;
+ transid = inode->root->last_trans;
defrag = kmem_cache_zalloc(btrfs_inode_defrag_cachep, GFP_NOFS);
if (!defrag)
return -ENOMEM;
- defrag->ino = btrfs_ino(BTRFS_I(inode));
+ defrag->ino = btrfs_ino(inode);
defrag->transid = transid;
defrag->root = root->root_key.objectid;
spin_lock(&fs_info->defrag_inodes_lock);
- if (!test_bit(BTRFS_INODE_IN_DEFRAG, &BTRFS_I(inode)->runtime_flags)) {
+ if (!test_bit(BTRFS_INODE_IN_DEFRAG, &inode->runtime_flags)) {
/*
* If we set IN_DEFRAG flag and evict the inode from memory,
* and then re-read this inode, this new inode doesn't have
* the same inode in the tree, we will merge them together (by
* __btrfs_add_inode_defrag()) and free the one that we want to requeue.
*/
-static void btrfs_requeue_inode_defrag(struct inode *inode,
+static void btrfs_requeue_inode_defrag(struct btrfs_inode *inode,
struct inode_defrag *defrag)
{
- struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
+ struct btrfs_fs_info *fs_info = btrfs_sb(inode->vfs_inode.i_sb);
int ret;
if (!__need_auto_defrag(fs_info))
*/
if (num_defrag == BTRFS_DEFRAG_BATCH) {
defrag->last_offset = range.start;
- btrfs_requeue_inode_defrag(inode, defrag);
+ btrfs_requeue_inode_defrag(BTRFS_I(inode), defrag);
} else if (defrag->last_offset && !defrag->cycled) {
/*
* we didn't fill our defrag batch, but
*/
defrag->last_offset = 0;
defrag->cycled = 1;
- btrfs_requeue_inode_defrag(inode, defrag);
+ btrfs_requeue_inode_defrag(BTRFS_I(inode), defrag);
} else {
kmem_cache_free(btrfs_inode_defrag_cachep, defrag);
}
* this drops all the extents in the cache that intersect the range
* [start, end]. Existing extents are split as required.
*/
-void btrfs_drop_extent_cache(struct inode *inode, u64 start, u64 end,
+void btrfs_drop_extent_cache(struct btrfs_inode *inode, u64 start, u64 end,
int skip_pinned)
{
struct extent_map *em;
struct extent_map *split = NULL;
struct extent_map *split2 = NULL;
- struct extent_map_tree *em_tree = &BTRFS_I(inode)->extent_tree;
+ struct extent_map_tree *em_tree = &inode->extent_tree;
u64 len = end - start + 1;
u64 gen;
int ret;
int leafs_visited = 0;
if (drop_cache)
- btrfs_drop_extent_cache(inode, start, end - 1, 0);
+ btrfs_drop_extent_cache(BTRFS_I(inode), start, end - 1, 0);
if (start >= BTRFS_I(inode)->disk_i_size && !replace_extent)
modify_tree = 0;
* two or three.
*/
int btrfs_mark_extent_written(struct btrfs_trans_handle *trans,
- struct inode *inode, u64 start, u64 end)
+ struct btrfs_inode *inode, u64 start, u64 end)
{
- struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
- struct btrfs_root *root = BTRFS_I(inode)->root;
+ struct btrfs_fs_info *fs_info = btrfs_sb(inode->vfs_inode.i_sb);
+ struct btrfs_root *root = inode->root;
struct extent_buffer *leaf;
struct btrfs_path *path;
struct btrfs_file_extent_item *fi;
int del_slot = 0;
int recow;
int ret;
- u64 ino = btrfs_ino(BTRFS_I(inode));
+ u64 ino = btrfs_ino(inode);
path = btrfs_alloc_path();
if (!path)
* the other < 0 number - Something wrong happens
*/
static noinline int
-lock_and_cleanup_extent_if_need(struct inode *inode, struct page **pages,
+lock_and_cleanup_extent_if_need(struct btrfs_inode *inode, struct page **pages,
size_t num_pages, loff_t pos,
size_t write_bytes,
u64 *lockstart, u64 *lockend,
struct extent_state **cached_state)
{
- struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
+ struct btrfs_fs_info *fs_info = btrfs_sb(inode->vfs_inode.i_sb);
u64 start_pos;
u64 last_pos;
int i;
+ round_up(pos + write_bytes - start_pos,
fs_info->sectorsize) - 1;
- if (start_pos < inode->i_size) {
+ if (start_pos < inode->vfs_inode.i_size) {
struct btrfs_ordered_extent *ordered;
- lock_extent_bits(&BTRFS_I(inode)->io_tree,
- start_pos, last_pos, cached_state);
+ lock_extent_bits(&inode->io_tree, start_pos, last_pos,
+ cached_state);
ordered = btrfs_lookup_ordered_range(inode, start_pos,
last_pos - start_pos + 1);
if (ordered &&
ordered->file_offset + ordered->len > start_pos &&
ordered->file_offset <= last_pos) {
- unlock_extent_cached(&BTRFS_I(inode)->io_tree,
- start_pos, last_pos,
- cached_state, GFP_NOFS);
+ unlock_extent_cached(&inode->io_tree, start_pos,
+ last_pos, cached_state, GFP_NOFS);
for (i = 0; i < num_pages; i++) {
unlock_page(pages[i]);
put_page(pages[i]);
}
- btrfs_start_ordered_extent(inode, ordered, 1);
+ btrfs_start_ordered_extent(&inode->vfs_inode,
+ ordered, 1);
btrfs_put_ordered_extent(ordered);
return -EAGAIN;
}
if (ordered)
btrfs_put_ordered_extent(ordered);
- clear_extent_bit(&BTRFS_I(inode)->io_tree, start_pos,
+ clear_extent_bit(&inode->io_tree, start_pos,
last_pos, EXTENT_DIRTY | EXTENT_DELALLOC |
EXTENT_DO_ACCOUNTING | EXTENT_DEFRAG,
0, 0, cached_state, GFP_NOFS);
return ret;
}
-static noinline int check_can_nocow(struct inode *inode, loff_t pos,
+static noinline int check_can_nocow(struct btrfs_inode *inode, loff_t pos,
size_t *write_bytes)
{
- struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
- struct btrfs_root *root = BTRFS_I(inode)->root;
+ struct btrfs_fs_info *fs_info = btrfs_sb(inode->vfs_inode.i_sb);
+ struct btrfs_root *root = inode->root;
struct btrfs_ordered_extent *ordered;
u64 lockstart, lockend;
u64 num_bytes;
fs_info->sectorsize) - 1;
while (1) {
- lock_extent(&BTRFS_I(inode)->io_tree, lockstart, lockend);
+ lock_extent(&inode->io_tree, lockstart, lockend);
ordered = btrfs_lookup_ordered_range(inode, lockstart,
lockend - lockstart + 1);
if (!ordered) {
break;
}
- unlock_extent(&BTRFS_I(inode)->io_tree, lockstart, lockend);
- btrfs_start_ordered_extent(inode, ordered, 1);
+ unlock_extent(&inode->io_tree, lockstart, lockend);
+ btrfs_start_ordered_extent(&inode->vfs_inode, ordered, 1);
btrfs_put_ordered_extent(ordered);
}
num_bytes = lockend - lockstart + 1;
- ret = can_nocow_extent(inode, lockstart, &num_bytes, NULL, NULL, NULL);
+ ret = can_nocow_extent(&inode->vfs_inode, lockstart, &num_bytes,
+ NULL, NULL, NULL);
if (ret <= 0) {
ret = 0;
btrfs_end_write_no_snapshoting(root);
num_bytes - pos + lockstart);
}
- unlock_extent(&BTRFS_I(inode)->io_tree, lockstart, lockend);
+ unlock_extent(&inode->io_tree, lockstart, lockend);
return ret;
}
if (ret < 0) {
if ((BTRFS_I(inode)->flags & (BTRFS_INODE_NODATACOW |
BTRFS_INODE_PREALLOC)) &&
- check_can_nocow(inode, pos, &write_bytes) > 0) {
+ check_can_nocow(BTRFS_I(inode), pos,
+ &write_bytes) > 0) {
/*
* For nodata cow case, no need to reserve
* data space.
}
}
- ret = btrfs_delalloc_reserve_metadata(inode, reserve_bytes);
+ ret = btrfs_delalloc_reserve_metadata(BTRFS_I(inode),
+ reserve_bytes);
if (ret) {
if (!only_release_metadata)
btrfs_free_reserved_data_space(inode, pos,
if (ret)
break;
- ret = lock_and_cleanup_extent_if_need(inode, pages, num_pages,
- pos, write_bytes, &lockstart,
- &lockend, &cached_state);
+ ret = lock_and_cleanup_extent_if_need(BTRFS_I(inode), pages,
+ num_pages, pos, write_bytes, &lockstart,
+ &lockend, &cached_state);
if (ret < 0) {
if (ret == -EAGAIN)
goto again;
spin_unlock(&BTRFS_I(inode)->lock);
}
if (only_release_metadata) {
- btrfs_delalloc_release_metadata(inode,
+ btrfs_delalloc_release_metadata(BTRFS_I(inode),
release_bytes);
} else {
u64 __pos;
if (release_bytes) {
if (only_release_metadata) {
btrfs_end_write_no_snapshoting(root);
- btrfs_delalloc_release_metadata(inode, release_bytes);
+ btrfs_delalloc_release_metadata(BTRFS_I(inode),
+ release_bytes);
} else {
btrfs_delalloc_release_space(inode,
round_down(pos, fs_info->sectorsize),
return 0;
}
-static int hole_mergeable(struct inode *inode, struct extent_buffer *leaf,
+static int hole_mergeable(struct btrfs_inode *inode, struct extent_buffer *leaf,
int slot, u64 start, u64 end)
{
struct btrfs_file_extent_item *fi;
return 0;
btrfs_item_key_to_cpu(leaf, &key, slot);
- if (key.objectid != btrfs_ino(BTRFS_I(inode)) ||
+ if (key.objectid != btrfs_ino(inode) ||
key.type != BTRFS_EXTENT_DATA_KEY)
return 0;
return 0;
}
-static int fill_holes(struct btrfs_trans_handle *trans, struct inode *inode,
- struct btrfs_path *path, u64 offset, u64 end)
+static int fill_holes(struct btrfs_trans_handle *trans,
+ struct btrfs_inode *inode,
+ struct btrfs_path *path, u64 offset, u64 end)
{
- struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
- struct btrfs_root *root = BTRFS_I(inode)->root;
+ struct btrfs_fs_info *fs_info = btrfs_sb(inode->vfs_inode.i_sb);
+ struct btrfs_root *root = inode->root;
struct extent_buffer *leaf;
struct btrfs_file_extent_item *fi;
struct extent_map *hole_em;
- struct extent_map_tree *em_tree = &BTRFS_I(inode)->extent_tree;
+ struct extent_map_tree *em_tree = &inode->extent_tree;
struct btrfs_key key;
int ret;
if (btrfs_fs_incompat(fs_info, NO_HOLES))
goto out;
- key.objectid = btrfs_ino(BTRFS_I(inode));
+ key.objectid = btrfs_ino(inode);
key.type = BTRFS_EXTENT_DATA_KEY;
key.offset = offset;
}
leaf = path->nodes[0];
- if (hole_mergeable(inode, leaf, path->slots[0]-1, offset, end)) {
+ if (hole_mergeable(inode, leaf, path->slots[0] - 1, offset, end)) {
u64 num_bytes;
path->slots[0]--;
}
btrfs_release_path(path);
- ret = btrfs_insert_file_extent(trans, root, btrfs_ino(BTRFS_I(inode)),
+ ret = btrfs_insert_file_extent(trans, root, btrfs_ino(inode),
offset, 0, 0, end - offset, 0, end - offset, 0, 0, 0);
if (ret)
return ret;
hole_em = alloc_extent_map();
if (!hole_em) {
btrfs_drop_extent_cache(inode, offset, end - 1, 0);
- set_bit(BTRFS_INODE_NEEDS_FULL_SYNC,
- &BTRFS_I(inode)->runtime_flags);
+ set_bit(BTRFS_INODE_NEEDS_FULL_SYNC, &inode->runtime_flags);
} else {
hole_em->start = offset;
hole_em->len = end - offset;
free_extent_map(hole_em);
if (ret)
set_bit(BTRFS_INODE_NEEDS_FULL_SYNC,
- &BTRFS_I(inode)->runtime_flags);
+ &inode->runtime_flags);
}
return 0;
struct extent_map *em;
int ret = 0;
- em = btrfs_get_extent(inode, NULL, 0, *start, *len, 0);
+ em = btrfs_get_extent(BTRFS_I(inode), NULL, 0, *start, *len, 0);
if (IS_ERR_OR_NULL(em)) {
if (!em)
ret = -ENOMEM;
trans->block_rsv = &fs_info->trans_block_rsv;
if (cur_offset < drop_end && cur_offset < ino_size) {
- ret = fill_holes(trans, inode, path, cur_offset,
- drop_end);
+ ret = fill_holes(trans, BTRFS_I(inode), path,
+ cur_offset, drop_end);
if (ret) {
/*
* If we failed then we didn't insert our hole
* cur_offset == drop_end).
*/
if (cur_offset < ino_size && cur_offset < drop_end) {
- ret = fill_holes(trans, inode, path, cur_offset, drop_end);
+ ret = fill_holes(trans, BTRFS_I(inode), path,
+ cur_offset, drop_end);
if (ret) {
/* Same comment as above. */
btrfs_abort_transaction(trans, ret);
*
* For qgroup space, it will be checked later.
*/
- ret = btrfs_alloc_data_chunk_ondemand(inode, alloc_end - alloc_start);
+ ret = btrfs_alloc_data_chunk_ondemand(BTRFS_I(inode),
+ alloc_end - alloc_start);
if (ret < 0)
return ret;
/* First, check if we exceed the qgroup limit */
INIT_LIST_HEAD(&reserve_list);
while (1) {
- em = btrfs_get_extent(inode, NULL, 0, cur_offset,
+ em = btrfs_get_extent(BTRFS_I(inode), NULL, 0, cur_offset,
alloc_end - cur_offset, 0);
if (IS_ERR_OR_NULL(em)) {
if (!em)
&cached_state);
while (start < inode->i_size) {
- em = btrfs_get_extent_fiemap(inode, NULL, 0, start, len, 0);
+ em = btrfs_get_extent_fiemap(BTRFS_I(inode), NULL, 0,
+ start, len, 0);
if (IS_ERR(em)) {
ret = PTR_ERR(em);
em = NULL;
#include <linux/pagemap.h>
#include <linux/sched.h>
+#include <linux/sched/signal.h>
#include <linux/slab.h>
#include <linux/math64.h>
#include <linux/ratelimit.h>
btrfs_free_path(path);
}
- btrfs_i_size_write(inode, 0);
+ btrfs_i_size_write(BTRFS_I(inode), 0);
truncate_pagecache(inode, 0);
/*
if (ret) {
if (release_metadata)
- btrfs_delalloc_release_metadata(inode, inode->i_size);
+ btrfs_delalloc_release_metadata(BTRFS_I(inode),
+ inode->i_size);
#ifdef DEBUG
btrfs_err(fs_info,
"failed to write free ino cache for root %llu",
ret = btrfs_prealloc_file_range_trans(inode, trans, 0, 0, prealloc,
prealloc, prealloc, &alloc_hint);
if (ret) {
- btrfs_delalloc_release_metadata(inode, prealloc);
+ btrfs_delalloc_release_metadata(BTRFS_I(inode), prealloc);
goto out_put;
}
}
set_bit(BTRFS_INODE_NEEDS_FULL_SYNC, &BTRFS_I(inode)->runtime_flags);
- btrfs_delalloc_release_metadata(inode, end + 1 - start);
- btrfs_drop_extent_cache(inode, start, aligned_end - 1, 0);
+ btrfs_delalloc_release_metadata(BTRFS_I(inode), end + 1 - start);
+ btrfs_drop_extent_cache(BTRFS_I(inode), start, aligned_end - 1, 0);
out:
/*
* Don't forget to free the reserved space, as for inlined extent
return 0;
}
-static inline void inode_should_defrag(struct inode *inode,
+static inline void inode_should_defrag(struct btrfs_inode *inode,
u64 start, u64 end, u64 num_bytes, u64 small_write)
{
/* If this is a small write inside eof, kick off a defrag */
if (num_bytes < small_write &&
- (start > 0 || end + 1 < BTRFS_I(inode)->disk_i_size))
+ (start > 0 || end + 1 < inode->disk_i_size))
btrfs_add_inode_defrag(NULL, inode);
}
int ret = 0;
struct page **pages = NULL;
unsigned long nr_pages;
- unsigned long nr_pages_ret = 0;
unsigned long total_compressed = 0;
unsigned long total_in = 0;
- unsigned long max_compressed = SZ_128K;
- unsigned long max_uncompressed = SZ_128K;
int i;
int will_compress;
int compress_type = fs_info->compress_type;
int redirty = 0;
- inode_should_defrag(inode, start, end, end - start + 1, SZ_16K);
+ inode_should_defrag(BTRFS_I(inode), start, end, end - start + 1,
+ SZ_16K);
actual_end = min_t(u64, isize, end + 1);
again:
will_compress = 0;
nr_pages = (end >> PAGE_SHIFT) - (start >> PAGE_SHIFT) + 1;
- nr_pages = min_t(unsigned long, nr_pages, SZ_128K / PAGE_SIZE);
+ BUILD_BUG_ON((BTRFS_MAX_COMPRESSED % PAGE_SIZE) != 0);
+ nr_pages = min_t(unsigned long, nr_pages,
+ BTRFS_MAX_COMPRESSED / PAGE_SIZE);
/*
* we don't want to send crud past the end of i_size through
(start > 0 || end + 1 < BTRFS_I(inode)->disk_i_size))
goto cleanup_and_bail_uncompressed;
- /* we want to make sure that amount of ram required to uncompress
- * an extent is reasonable, so we limit the total size in ram
- * of a compressed extent to 128k. This is a crucial number
- * because it also controls how easily we can spread reads across
- * cpus for decompression.
- *
- * We also want to make sure the amount of IO required to do
- * a random read is reasonably small, so we limit the size of
- * a compressed extent to 128k.
- */
- total_compressed = min(total_compressed, max_uncompressed);
+ total_compressed = min_t(unsigned long, total_compressed,
+ BTRFS_MAX_UNCOMPRESSED);
num_bytes = ALIGN(end - start + 1, blocksize);
num_bytes = max(blocksize, num_bytes);
total_in = 0;
redirty = 1;
ret = btrfs_compress_pages(compress_type,
inode->i_mapping, start,
- total_compressed, pages,
- nr_pages, &nr_pages_ret,
+ pages,
+ &nr_pages,
&total_in,
- &total_compressed,
- max_compressed);
+ &total_compressed);
if (!ret) {
unsigned long offset = total_compressed &
(PAGE_SIZE - 1);
- struct page *page = pages[nr_pages_ret - 1];
+ struct page *page = pages[nr_pages - 1];
char *kaddr;
/* zero the tail end of the last page, we might be
* will submit them to the elevator.
*/
add_async_extent(async_cow, start, num_bytes,
- total_compressed, pages, nr_pages_ret,
+ total_compressed, pages, nr_pages,
compress_type);
if (start + num_bytes < end) {
* the compression code ran but failed to make things smaller,
* free any pages it allocated and our page pointer array
*/
- for (i = 0; i < nr_pages_ret; i++) {
+ for (i = 0; i < nr_pages; i++) {
WARN_ON(pages[i]->mapping);
put_page(pages[i]);
}
kfree(pages);
pages = NULL;
total_compressed = 0;
- nr_pages_ret = 0;
+ nr_pages = 0;
/* flag the file so we don't compress in the future */
if (!btrfs_test_opt(fs_info, FORCE_COMPRESS) &&
return;
free_pages_out:
- for (i = 0; i < nr_pages_ret; i++) {
+ for (i = 0; i < nr_pages; i++) {
WARN_ON(pages[i]->mapping);
put_page(pages[i]);
}
BTRFS_ORDERED_COMPRESSED,
async_extent->compress_type);
if (ret) {
- btrfs_drop_extent_cache(inode, async_extent->start,
+ btrfs_drop_extent_cache(BTRFS_I(inode),
+ async_extent->start,
async_extent->start +
async_extent->ram_size - 1, 0);
goto out_free_reserve;
struct extent_map *em;
int ret = 0;
- if (btrfs_is_free_space_inode(inode)) {
+ if (btrfs_is_free_space_inode(BTRFS_I(inode))) {
WARN_ON_ONCE(1);
ret = -EINVAL;
goto out_unlock;
num_bytes = max(blocksize, num_bytes);
disk_num_bytes = num_bytes;
- inode_should_defrag(inode, start, end, num_bytes, SZ_64K);
+ inode_should_defrag(BTRFS_I(inode), start, end, num_bytes, SZ_64K);
if (start == 0) {
/* lets try to make an inline extent */
btrfs_super_total_bytes(fs_info->super_copy));
alloc_hint = get_extent_allocation_hint(inode, start, num_bytes);
- btrfs_drop_extent_cache(inode, start, start + num_bytes - 1, 0);
+ btrfs_drop_extent_cache(BTRFS_I(inode), start,
+ start + num_bytes - 1, 0);
while (disk_num_bytes > 0) {
unsigned long op;
return ret;
out_drop_extent_cache:
- btrfs_drop_extent_cache(inode, start, start + ram_size - 1, 0);
+ btrfs_drop_extent_cache(BTRFS_I(inode), start, start + ram_size - 1, 0);
out_reserve:
btrfs_dec_block_group_reservations(fs_info, ins.objectid);
btrfs_free_reserved_extent(fs_info, ins.objectid, ins.offset, 1);
return -ENOMEM;
}
- nolock = btrfs_is_free_space_inode(inode);
+ nolock = btrfs_is_free_space_inode(BTRFS_I(inode));
cow_start = (u64)-1;
cur_offset = start;
* either valid or do not exist.
*/
if (csum_exist_in_range(fs_info, disk_bytenr,
- num_bytes))
+ num_bytes)) {
+ if (!nolock)
+ btrfs_end_write_no_snapshoting(root);
goto out_check;
- if (!btrfs_inc_nocow_writers(fs_info, disk_bytenr))
+ }
+ if (!btrfs_inc_nocow_writers(fs_info, disk_bytenr)) {
+ if (!nolock)
+ btrfs_end_write_no_snapshoting(root);
goto out_check;
+ }
nocow = 1;
} else if (extent_type == BTRFS_FILE_EXTENT_INLINE) {
extent_end = found_key.offset +
}
static void btrfs_del_delalloc_inode(struct btrfs_root *root,
- struct inode *inode)
+ struct btrfs_inode *inode)
{
- struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
+ struct btrfs_fs_info *fs_info = btrfs_sb(inode->vfs_inode.i_sb);
spin_lock(&root->delalloc_lock);
- if (!list_empty(&BTRFS_I(inode)->delalloc_inodes)) {
- list_del_init(&BTRFS_I(inode)->delalloc_inodes);
+ if (!list_empty(&inode->delalloc_inodes)) {
+ list_del_init(&inode->delalloc_inodes);
clear_bit(BTRFS_INODE_IN_DELALLOC_LIST,
- &BTRFS_I(inode)->runtime_flags);
+ &inode->runtime_flags);
root->nr_delalloc_inodes--;
if (!root->nr_delalloc_inodes) {
spin_lock(&fs_info->delalloc_root_lock);
if (!(state->state & EXTENT_DELALLOC) && (*bits & EXTENT_DELALLOC)) {
struct btrfs_root *root = BTRFS_I(inode)->root;
u64 len = state->end + 1 - state->start;
- bool do_list = !btrfs_is_free_space_inode(inode);
+ bool do_list = !btrfs_is_free_space_inode(BTRFS_I(inode));
if (*bits & EXTENT_FIRST_DELALLOC) {
*bits &= ~EXTENT_FIRST_DELALLOC;
/*
* extent_io.c clear_bit_hook, see set_bit_hook for why
*/
-static void btrfs_clear_bit_hook(struct inode *inode,
+static void btrfs_clear_bit_hook(struct btrfs_inode *inode,
struct extent_state *state,
unsigned *bits)
{
- struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
+ struct btrfs_fs_info *fs_info = btrfs_sb(inode->vfs_inode.i_sb);
u64 len = state->end + 1 - state->start;
u32 num_extents = count_max_extents(len);
- spin_lock(&BTRFS_I(inode)->lock);
+ spin_lock(&inode->lock);
if ((state->state & EXTENT_DEFRAG) && (*bits & EXTENT_DEFRAG))
- BTRFS_I(inode)->defrag_bytes -= len;
- spin_unlock(&BTRFS_I(inode)->lock);
+ inode->defrag_bytes -= len;
+ spin_unlock(&inode->lock);
/*
* set_bit and clear bit hooks normally require _irqsave/restore
* bit, which is only set or cleared with irqs on
*/
if ((state->state & EXTENT_DELALLOC) && (*bits & EXTENT_DELALLOC)) {
- struct btrfs_root *root = BTRFS_I(inode)->root;
+ struct btrfs_root *root = inode->root;
bool do_list = !btrfs_is_free_space_inode(inode);
if (*bits & EXTENT_FIRST_DELALLOC) {
*bits &= ~EXTENT_FIRST_DELALLOC;
} else if (!(*bits & EXTENT_DO_ACCOUNTING)) {
- spin_lock(&BTRFS_I(inode)->lock);
- BTRFS_I(inode)->outstanding_extents -= num_extents;
- spin_unlock(&BTRFS_I(inode)->lock);
+ spin_lock(&inode->lock);
+ inode->outstanding_extents -= num_extents;
+ spin_unlock(&inode->lock);
}
/*
&& do_list && !(state->state & EXTENT_NORESERVE)
&& (*bits & (EXTENT_DO_ACCOUNTING |
EXTENT_CLEAR_DATA_RESV)))
- btrfs_free_reserved_data_space_noquota(inode,
+ btrfs_free_reserved_data_space_noquota(
+ &inode->vfs_inode,
state->start, len);
__percpu_counter_add(&fs_info->delalloc_bytes, -len,
fs_info->delalloc_batch);
- spin_lock(&BTRFS_I(inode)->lock);
- BTRFS_I(inode)->delalloc_bytes -= len;
- if (do_list && BTRFS_I(inode)->delalloc_bytes == 0 &&
+ spin_lock(&inode->lock);
+ inode->delalloc_bytes -= len;
+ if (do_list && inode->delalloc_bytes == 0 &&
test_bit(BTRFS_INODE_IN_DELALLOC_LIST,
- &BTRFS_I(inode)->runtime_flags))
+ &inode->runtime_flags))
btrfs_del_delalloc_inode(root, inode);
- spin_unlock(&BTRFS_I(inode)->lock);
+ spin_unlock(&inode->lock);
}
}
skip_sum = BTRFS_I(inode)->flags & BTRFS_INODE_NODATASUM;
- if (btrfs_is_free_space_inode(inode))
+ if (btrfs_is_free_space_inode(BTRFS_I(inode)))
metadata = BTRFS_WQ_ENDIO_FREE_SPACE;
if (bio_op(bio) != REQ_OP_WRITE) {
if (PagePrivate2(page))
goto out;
- ordered = btrfs_lookup_ordered_range(inode, page_start,
+ ordered = btrfs_lookup_ordered_range(BTRFS_I(inode), page_start,
PAGE_SIZE);
if (ordered) {
unlock_extent_cached(&BTRFS_I(inode)->io_tree, page_start,
bool nolock;
bool truncated = false;
- nolock = btrfs_is_free_space_inode(inode);
+ nolock = btrfs_is_free_space_inode(BTRFS_I(inode));
if (test_bit(BTRFS_ORDERED_IOERR, &ordered_extent->flags)) {
ret = -EIO;
goto out;
}
- btrfs_free_io_failure_record(inode, ordered_extent->file_offset,
- ordered_extent->file_offset +
- ordered_extent->len - 1);
+ btrfs_free_io_failure_record(BTRFS_I(inode),
+ ordered_extent->file_offset,
+ ordered_extent->file_offset +
+ ordered_extent->len - 1);
if (test_bit(BTRFS_ORDERED_TRUNCATED, &ordered_extent->flags)) {
truncated = true;
compress_type = ordered_extent->compress_type;
if (test_bit(BTRFS_ORDERED_PREALLOC, &ordered_extent->flags)) {
BUG_ON(compress_type);
- ret = btrfs_mark_extent_written(trans, inode,
+ ret = btrfs_mark_extent_written(trans, BTRFS_I(inode),
ordered_extent->file_offset,
ordered_extent->file_offset +
logical_len);
ordered_extent->len - 1, &cached_state, GFP_NOFS);
out:
if (root != fs_info->tree_root)
- btrfs_delalloc_release_metadata(inode, ordered_extent->len);
+ btrfs_delalloc_release_metadata(BTRFS_I(inode),
+ ordered_extent->len);
if (trans)
btrfs_end_transaction(trans);
clear_extent_uptodate(io_tree, start, end, NULL, GFP_NOFS);
/* Drop the cache for the part of the extent we didn't write. */
- btrfs_drop_extent_cache(inode, start, end, 0);
+ btrfs_drop_extent_cache(BTRFS_I(inode), start, end, 0);
/*
* If the ordered extent had an IOERR or something else went
btrfs_finish_ordered_io(ordered_extent);
}
-static int btrfs_writepage_end_io_hook(struct page *page, u64 start, u64 end,
+static void btrfs_writepage_end_io_hook(struct page *page, u64 start, u64 end,
struct extent_state *state, int uptodate)
{
struct inode *inode = page->mapping->host;
ClearPagePrivate2(page);
if (!btrfs_dec_test_ordered_pending(inode, &ordered_extent, start,
end - start + 1, uptodate))
- return 0;
+ return;
- if (btrfs_is_free_space_inode(inode)) {
+ if (btrfs_is_free_space_inode(BTRFS_I(inode))) {
wq = fs_info->endio_freespace_worker;
func = btrfs_freespace_write_helper;
} else {
btrfs_init_work(&ordered_extent->work, func, finish_ordered_fn, NULL,
NULL);
btrfs_queue_work(wq, &ordered_extent->work);
-
- return 0;
}
static int __readpage_endio_check(struct inode *inode,
kunmap_atomic(kaddr);
return 0;
zeroit:
- btrfs_print_data_csum_error(inode, start, csum, csum_expected,
+ btrfs_print_data_csum_error(BTRFS_I(inode), start, csum, csum_expected,
io_bio->mirror_num);
memset(kaddr + pgoff, 1, len);
flush_dcache_page(page);
* NOTE: caller of this function should reserve 5 units of metadata for
* this function.
*/
-int btrfs_orphan_add(struct btrfs_trans_handle *trans, struct inode *inode)
+int btrfs_orphan_add(struct btrfs_trans_handle *trans,
+ struct btrfs_inode *inode)
{
- struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
- struct btrfs_root *root = BTRFS_I(inode)->root;
+ struct btrfs_fs_info *fs_info = btrfs_sb(inode->vfs_inode.i_sb);
+ struct btrfs_root *root = inode->root;
struct btrfs_block_rsv *block_rsv = NULL;
int reserve = 0;
int insert = 0;
}
if (!test_and_set_bit(BTRFS_INODE_HAS_ORPHAN_ITEM,
- &BTRFS_I(inode)->runtime_flags)) {
+ &inode->runtime_flags)) {
#if 0
/*
* For proper ENOSPC handling, we should do orphan
}
if (!test_and_set_bit(BTRFS_INODE_ORPHAN_META_RESERVED,
- &BTRFS_I(inode)->runtime_flags))
+ &inode->runtime_flags))
reserve = 1;
spin_unlock(&root->orphan_lock);
if (ret) {
atomic_dec(&root->orphan_inodes);
clear_bit(BTRFS_INODE_ORPHAN_META_RESERVED,
- &BTRFS_I(inode)->runtime_flags);
+ &inode->runtime_flags);
if (insert)
clear_bit(BTRFS_INODE_HAS_ORPHAN_ITEM,
- &BTRFS_I(inode)->runtime_flags);
+ &inode->runtime_flags);
return ret;
}
}
/* insert an orphan item to track this unlinked/truncated file */
if (insert >= 1) {
- ret = btrfs_insert_orphan_item(trans, root,
- btrfs_ino(BTRFS_I(inode)));
+ ret = btrfs_insert_orphan_item(trans, root, btrfs_ino(inode));
if (ret) {
atomic_dec(&root->orphan_inodes);
if (reserve) {
clear_bit(BTRFS_INODE_ORPHAN_META_RESERVED,
- &BTRFS_I(inode)->runtime_flags);
+ &inode->runtime_flags);
btrfs_orphan_release_metadata(inode);
}
if (ret != -EEXIST) {
clear_bit(BTRFS_INODE_HAS_ORPHAN_ITEM,
- &BTRFS_I(inode)->runtime_flags);
+ &inode->runtime_flags);
btrfs_abort_transaction(trans, ret);
return ret;
}
* item for this particular inode.
*/
static int btrfs_orphan_del(struct btrfs_trans_handle *trans,
- struct inode *inode)
+ struct btrfs_inode *inode)
{
- struct btrfs_root *root = BTRFS_I(inode)->root;
+ struct btrfs_root *root = inode->root;
int delete_item = 0;
int release_rsv = 0;
int ret = 0;
spin_lock(&root->orphan_lock);
if (test_and_clear_bit(BTRFS_INODE_HAS_ORPHAN_ITEM,
- &BTRFS_I(inode)->runtime_flags))
+ &inode->runtime_flags))
delete_item = 1;
if (test_and_clear_bit(BTRFS_INODE_ORPHAN_META_RESERVED,
- &BTRFS_I(inode)->runtime_flags))
+ &inode->runtime_flags))
release_rsv = 1;
spin_unlock(&root->orphan_lock);
atomic_dec(&root->orphan_inodes);
if (trans)
ret = btrfs_del_orphan_item(trans, root,
- btrfs_ino(BTRFS_I(inode)));
+ btrfs_ino(inode));
}
if (release_rsv)
ret = PTR_ERR(trans);
goto out;
}
- ret = btrfs_orphan_add(trans, inode);
+ ret = btrfs_orphan_add(trans, BTRFS_I(inode));
btrfs_end_transaction(trans);
if (ret) {
iput(inode);
ret = btrfs_truncate(inode);
if (ret)
- btrfs_orphan_del(NULL, inode);
+ btrfs_orphan_del(NULL, BTRFS_I(inode));
} else {
nr_unlink++;
}
set_nlink(inode, btrfs_inode_nlink(leaf, inode_item));
i_uid_write(inode, btrfs_inode_uid(leaf, inode_item));
i_gid_write(inode, btrfs_inode_gid(leaf, inode_item));
- btrfs_i_size_write(inode, btrfs_inode_size(leaf, inode_item));
+ btrfs_i_size_write(BTRFS_I(inode), btrfs_inode_size(leaf, inode_item));
inode->i_atime.tv_sec = btrfs_timespec_sec(leaf, &inode_item->atime);
inode->i_atime.tv_nsec = btrfs_timespec_nsec(leaf, &inode_item->atime);
* The data relocation inode should also be directly updated
* without delay
*/
- if (!btrfs_is_free_space_inode(inode)
+ if (!btrfs_is_free_space_inode(BTRFS_I(inode))
&& root->root_key.objectid != BTRFS_DATA_RELOC_TREE_OBJECTID
&& !test_bit(BTRFS_FS_LOG_RECOVERING, &fs_info->flags)) {
btrfs_update_root_times(trans, root);
if (ret)
goto out;
- btrfs_i_size_write(&dir->vfs_inode,
- dir->vfs_inode.i_size - name_len * 2);
+ btrfs_i_size_write(dir, dir->vfs_inode.i_size - name_len * 2);
inode_inc_iversion(&inode->vfs_inode);
inode_inc_iversion(&dir->vfs_inode);
inode->vfs_inode.i_ctime = dir->vfs_inode.i_mtime =
goto out;
if (inode->i_nlink == 0) {
- ret = btrfs_orphan_add(trans, inode);
+ ret = btrfs_orphan_add(trans, BTRFS_I(inode));
if (ret)
goto out;
}
goto out;
}
- btrfs_i_size_write(dir, dir->i_size - name_len * 2);
+ btrfs_i_size_write(BTRFS_I(dir), dir->i_size - name_len * 2);
inode_inc_iversion(dir);
dir->i_mtime = dir->i_ctime = current_time(dir);
ret = btrfs_update_inode_fallback(trans, root, dir);
goto out;
}
- err = btrfs_orphan_add(trans, inode);
+ err = btrfs_orphan_add(trans, BTRFS_I(inode));
if (err)
goto out;
BTRFS_I(d_inode(dentry)), dentry->d_name.name,
dentry->d_name.len);
if (!err) {
- btrfs_i_size_write(inode, 0);
+ btrfs_i_size_write(BTRFS_I(inode), 0);
/*
* Propagate the last_unlink_trans value of the deleted dir to
* its parent directory. This is to prevent an unrecoverable
* for non-free space inodes and ref cows, we want to back off from
* time to time
*/
- if (!btrfs_is_free_space_inode(inode) &&
+ if (!btrfs_is_free_space_inode(BTRFS_I(inode)) &&
test_bit(BTRFS_ROOT_REF_COWS, &root->state))
be_nice = 1;
*/
if (test_bit(BTRFS_ROOT_REF_COWS, &root->state) ||
root == fs_info->tree_root)
- btrfs_drop_extent_cache(inode, ALIGN(new_size,
+ btrfs_drop_extent_cache(BTRFS_I(inode), ALIGN(new_size,
fs_info->sectorsize),
(u64)-1, 0);
if (found_type > min_type) {
del_item = 1;
} else {
- if (item_end < new_size) {
- /*
- * With NO_HOLES mode, for the following mapping
- *
- * [0-4k][hole][8k-12k]
- *
- * if truncating isize down to 6k, it ends up
- * isize being 8k.
- */
- if (btrfs_fs_incompat(root->fs_info, NO_HOLES))
- last_size = new_size;
+ if (item_end < new_size)
break;
- }
if (found_key.offset >= new_size)
del_item = 1;
else
btrfs_abort_transaction(trans, ret);
}
error:
- if (root->root_key.objectid != BTRFS_TREE_LOG_OBJECTID)
+ if (root->root_key.objectid != BTRFS_TREE_LOG_OBJECTID) {
+ ASSERT(last_size >= new_size);
+ if (!err && last_size > new_size)
+ last_size = new_size;
btrfs_ordered_update_i_size(inode, last_size, NULL);
+ }
btrfs_free_path(path);
lock_extent_bits(io_tree, hole_start, block_end - 1,
&cached_state);
- ordered = btrfs_lookup_ordered_range(inode, hole_start,
+ ordered = btrfs_lookup_ordered_range(BTRFS_I(inode), hole_start,
block_end - hole_start);
if (!ordered)
break;
cur_offset = hole_start;
while (1) {
- em = btrfs_get_extent(inode, NULL, 0, cur_offset,
+ em = btrfs_get_extent(BTRFS_I(inode), NULL, 0, cur_offset,
block_end - cur_offset, 0);
if (IS_ERR(em)) {
err = PTR_ERR(em);
hole_size);
if (err)
break;
- btrfs_drop_extent_cache(inode, cur_offset,
+ btrfs_drop_extent_cache(BTRFS_I(inode), cur_offset,
cur_offset + hole_size - 1, 0);
hole_em = alloc_extent_map();
if (!hole_em) {
write_unlock(&em_tree->lock);
if (err != -EEXIST)
break;
- btrfs_drop_extent_cache(inode, cur_offset,
+ btrfs_drop_extent_cache(BTRFS_I(inode),
+ cur_offset,
cur_offset +
hole_size - 1, 0);
}
* so we need to guarantee from this point on that everything
* will be consistent.
*/
- ret = btrfs_orphan_add(trans, inode);
+ ret = btrfs_orphan_add(trans, BTRFS_I(inode));
btrfs_end_transaction(trans);
if (ret)
return ret;
truncate_setsize(inode, newsize);
/* Disable nonlocked read DIO to avoid the end less truncate */
- btrfs_inode_block_unlocked_dio(inode);
+ btrfs_inode_block_unlocked_dio(BTRFS_I(inode));
inode_dio_wait(inode);
- btrfs_inode_resume_unlocked_dio(inode);
+ btrfs_inode_resume_unlocked_dio(BTRFS_I(inode));
ret = btrfs_truncate(inode);
if (ret && inode->i_nlink) {
/* To get a stable disk_i_size */
err = btrfs_wait_ordered_range(inode, 0, (u64)-1);
if (err) {
- btrfs_orphan_del(NULL, inode);
+ btrfs_orphan_del(NULL, BTRFS_I(inode));
return err;
}
*/
trans = btrfs_join_transaction(root);
if (IS_ERR(trans)) {
- btrfs_orphan_del(NULL, inode);
+ btrfs_orphan_del(NULL, BTRFS_I(inode));
return ret;
}
i_size_write(inode, BTRFS_I(inode)->disk_i_size);
- err = btrfs_orphan_del(trans, inode);
+ err = btrfs_orphan_del(trans, BTRFS_I(inode));
if (err)
btrfs_abort_transaction(trans, err);
btrfs_end_transaction(trans);
if (inode->i_nlink &&
((btrfs_root_refs(&root->root_item) != 0 &&
root->root_key.objectid != BTRFS_ROOT_TREE_OBJECTID) ||
- btrfs_is_free_space_inode(inode)))
+ btrfs_is_free_space_inode(BTRFS_I(inode))))
goto no_delete;
if (is_bad_inode(inode)) {
- btrfs_orphan_del(NULL, inode);
+ btrfs_orphan_del(NULL, BTRFS_I(inode));
goto no_delete;
}
/* do we really want it for ->i_nlink > 0 and zero btrfs_root_refs? */
if (!special_file(inode->i_mode))
btrfs_wait_ordered_range(inode, 0, (u64)-1);
- btrfs_free_io_failure_record(inode, 0, (u64)-1);
+ btrfs_free_io_failure_record(BTRFS_I(inode), 0, (u64)-1);
if (test_bit(BTRFS_FS_LOG_RECOVERING, &fs_info->flags)) {
BUG_ON(test_bit(BTRFS_INODE_HAS_ORPHAN_ITEM,
ret = btrfs_commit_inode_delayed_inode(BTRFS_I(inode));
if (ret) {
- btrfs_orphan_del(NULL, inode);
+ btrfs_orphan_del(NULL, BTRFS_I(inode));
goto no_delete;
}
rsv = btrfs_alloc_block_rsv(fs_info, BTRFS_BLOCK_RSV_TEMP);
if (!rsv) {
- btrfs_orphan_del(NULL, inode);
+ btrfs_orphan_del(NULL, BTRFS_I(inode));
goto no_delete;
}
rsv->size = min_size;
rsv->failfast = 1;
global_rsv = &fs_info->global_block_rsv;
- btrfs_i_size_write(inode, 0);
+ btrfs_i_size_write(BTRFS_I(inode), 0);
/*
* This is a bit simpler than btrfs_truncate since we've already
btrfs_warn(fs_info,
"Could not get space for a delete, will truncate on mount %d",
ret);
- btrfs_orphan_del(NULL, inode);
+ btrfs_orphan_del(NULL, BTRFS_I(inode));
btrfs_free_block_rsv(fs_info, rsv);
goto no_delete;
}
trans = btrfs_join_transaction(root);
if (IS_ERR(trans)) {
- btrfs_orphan_del(NULL, inode);
+ btrfs_orphan_del(NULL, BTRFS_I(inode));
btrfs_free_block_rsv(fs_info, rsv);
goto no_delete;
}
if (ret) {
ret = btrfs_commit_transaction(trans);
if (ret) {
- btrfs_orphan_del(NULL, inode);
+ btrfs_orphan_del(NULL, BTRFS_I(inode));
btrfs_free_block_rsv(fs_info, rsv);
goto no_delete;
}
*/
if (ret == 0) {
trans->block_rsv = root->orphan_block_rsv;
- btrfs_orphan_del(trans, inode);
+ btrfs_orphan_del(trans, BTRFS_I(inode));
} else {
- btrfs_orphan_del(NULL, inode);
+ btrfs_orphan_del(NULL, BTRFS_I(inode));
}
trans->block_rsv = &fs_info->trans_block_rsv;
if (test_bit(BTRFS_INODE_DUMMY, &BTRFS_I(inode)->runtime_flags))
return 0;
- if (btrfs_fs_closing(root->fs_info) && btrfs_is_free_space_inode(inode))
+ if (btrfs_fs_closing(root->fs_info) &&
+ btrfs_is_free_space_inode(BTRFS_I(inode)))
nolock = true;
if (wbc->sync_mode == WB_SYNC_ALL) {
* and then set the in-memory index_cnt variable to reflect
* free sequence numbers
*/
-static int btrfs_set_inode_index_count(struct inode *inode)
+static int btrfs_set_inode_index_count(struct btrfs_inode *inode)
{
- struct btrfs_root *root = BTRFS_I(inode)->root;
+ struct btrfs_root *root = inode->root;
struct btrfs_key key, found_key;
struct btrfs_path *path;
struct extent_buffer *leaf;
int ret;
- key.objectid = btrfs_ino(BTRFS_I(inode));
+ key.objectid = btrfs_ino(inode);
key.type = BTRFS_DIR_INDEX_KEY;
key.offset = (u64)-1;
* else has to start at 2
*/
if (path->slots[0] == 0) {
- BTRFS_I(inode)->index_cnt = 2;
+ inode->index_cnt = 2;
goto out;
}
leaf = path->nodes[0];
btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]);
- if (found_key.objectid != btrfs_ino(BTRFS_I(inode)) ||
+ if (found_key.objectid != btrfs_ino(inode) ||
found_key.type != BTRFS_DIR_INDEX_KEY) {
- BTRFS_I(inode)->index_cnt = 2;
+ inode->index_cnt = 2;
goto out;
}
- BTRFS_I(inode)->index_cnt = found_key.offset + 1;
+ inode->index_cnt = found_key.offset + 1;
out:
btrfs_free_path(path);
return ret;
* helper to find a free sequence number in a given directory. This current
* code is very simple, later versions will do smarter things in the btree
*/
-int btrfs_set_inode_index(struct inode *dir, u64 *index)
+int btrfs_set_inode_index(struct btrfs_inode *dir, u64 *index)
{
int ret = 0;
- if (BTRFS_I(dir)->index_cnt == (u64)-1) {
- ret = btrfs_inode_delayed_dir_index_count(BTRFS_I(dir));
+ if (dir->index_cnt == (u64)-1) {
+ ret = btrfs_inode_delayed_dir_index_count(dir);
if (ret) {
ret = btrfs_set_inode_index_count(dir);
if (ret)
}
}
- *index = BTRFS_I(dir)->index_cnt;
- BTRFS_I(dir)->index_cnt++;
+ *index = dir->index_cnt;
+ dir->index_cnt++;
return ret;
}
if (dir && name) {
trace_btrfs_inode_request(dir);
- ret = btrfs_set_inode_index(dir, index);
+ ret = btrfs_set_inode_index(BTRFS_I(dir), index);
if (ret) {
btrfs_free_path(path);
iput(inode);
* inode to the parent directory.
*/
int btrfs_add_link(struct btrfs_trans_handle *trans,
- struct inode *parent_inode, struct inode *inode,
+ struct btrfs_inode *parent_inode, struct btrfs_inode *inode,
const char *name, int name_len, int add_backref, u64 index)
{
- struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
+ struct btrfs_fs_info *fs_info = btrfs_sb(inode->vfs_inode.i_sb);
int ret = 0;
struct btrfs_key key;
- struct btrfs_root *root = BTRFS_I(parent_inode)->root;
- u64 ino = btrfs_ino(BTRFS_I(inode));
- u64 parent_ino = btrfs_ino(BTRFS_I(parent_inode));
+ struct btrfs_root *root = parent_inode->root;
+ u64 ino = btrfs_ino(inode);
+ u64 parent_ino = btrfs_ino(parent_inode);
if (unlikely(ino == BTRFS_FIRST_FREE_OBJECTID)) {
- memcpy(&key, &BTRFS_I(inode)->root->root_key, sizeof(key));
+ memcpy(&key, &inode->root->root_key, sizeof(key));
} else {
key.objectid = ino;
key.type = BTRFS_INODE_ITEM_KEY;
ret = btrfs_insert_dir_item(trans, root, name, name_len,
parent_inode, &key,
- btrfs_inode_type(inode), index);
+ btrfs_inode_type(&inode->vfs_inode), index);
if (ret == -EEXIST || ret == -EOVERFLOW)
goto fail_dir_item;
else if (ret) {
return ret;
}
- btrfs_i_size_write(parent_inode, parent_inode->i_size +
+ btrfs_i_size_write(parent_inode, parent_inode->vfs_inode.i_size +
name_len * 2);
- inode_inc_iversion(parent_inode);
- parent_inode->i_mtime = parent_inode->i_ctime =
- current_time(parent_inode);
- ret = btrfs_update_inode(trans, root, parent_inode);
+ inode_inc_iversion(&parent_inode->vfs_inode);
+ parent_inode->vfs_inode.i_mtime = parent_inode->vfs_inode.i_ctime =
+ current_time(&parent_inode->vfs_inode);
+ ret = btrfs_update_inode(trans, root, &parent_inode->vfs_inode);
if (ret)
btrfs_abort_transaction(trans, ret);
return ret;
}
static int btrfs_add_nondir(struct btrfs_trans_handle *trans,
- struct inode *dir, struct dentry *dentry,
- struct inode *inode, int backref, u64 index)
+ struct btrfs_inode *dir, struct dentry *dentry,
+ struct btrfs_inode *inode, int backref, u64 index)
{
int err = btrfs_add_link(trans, dir, inode,
dentry->d_name.name, dentry->d_name.len,
if (err)
goto out_unlock_inode;
- err = btrfs_add_nondir(trans, dir, dentry, inode, 0, index);
+ err = btrfs_add_nondir(trans, BTRFS_I(dir), dentry, BTRFS_I(inode),
+ 0, index);
if (err) {
goto out_unlock_inode;
} else {
if (err)
goto out_unlock_inode;
- err = btrfs_add_nondir(trans, dir, dentry, inode, 0, index);
+ err = btrfs_add_nondir(trans, BTRFS_I(dir), dentry, BTRFS_I(inode),
+ 0, index);
if (err)
goto out_unlock_inode;
if (inode->i_nlink >= BTRFS_LINK_MAX)
return -EMLINK;
- err = btrfs_set_inode_index(dir, &index);
+ err = btrfs_set_inode_index(BTRFS_I(dir), &index);
if (err)
goto fail;
ihold(inode);
set_bit(BTRFS_INODE_COPY_EVERYTHING, &BTRFS_I(inode)->runtime_flags);
- err = btrfs_add_nondir(trans, dir, dentry, inode, 1, index);
+ err = btrfs_add_nondir(trans, BTRFS_I(dir), dentry, BTRFS_I(inode),
+ 1, index);
if (err) {
drop_inode = 1;
* If new hard link count is 1, it's a file created
* with open(2) O_TMPFILE flag.
*/
- err = btrfs_orphan_del(trans, inode);
+ err = btrfs_orphan_del(trans, BTRFS_I(inode));
if (err)
goto fail;
}
if (err)
goto out_fail_inode;
- btrfs_i_size_write(inode, 0);
+ btrfs_i_size_write(BTRFS_I(inode), 0);
err = btrfs_update_inode(trans, root, inode);
if (err)
goto out_fail_inode;
- err = btrfs_add_link(trans, dir, inode, dentry->d_name.name,
- dentry->d_name.len, 0, index);
+ err = btrfs_add_link(trans, BTRFS_I(dir), BTRFS_I(inode),
+ dentry->d_name.name,
+ dentry->d_name.len, 0, index);
if (err)
goto out_fail_inode;
* This also copies inline extents directly into the page.
*/
-struct extent_map *btrfs_get_extent(struct inode *inode, struct page *page,
- size_t pg_offset, u64 start, u64 len,
- int create)
+struct extent_map *btrfs_get_extent(struct btrfs_inode *inode,
+ struct page *page,
+ size_t pg_offset, u64 start, u64 len,
+ int create)
{
- struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
+ struct btrfs_fs_info *fs_info = btrfs_sb(inode->vfs_inode.i_sb);
int ret;
int err = 0;
u64 extent_start = 0;
u64 extent_end = 0;
- u64 objectid = btrfs_ino(BTRFS_I(inode));
+ u64 objectid = btrfs_ino(inode);
u32 found_type;
struct btrfs_path *path = NULL;
- struct btrfs_root *root = BTRFS_I(inode)->root;
+ struct btrfs_root *root = inode->root;
struct btrfs_file_extent_item *item;
struct extent_buffer *leaf;
struct btrfs_key found_key;
struct extent_map *em = NULL;
- struct extent_map_tree *em_tree = &BTRFS_I(inode)->extent_tree;
- struct extent_io_tree *io_tree = &BTRFS_I(inode)->io_tree;
+ struct extent_map_tree *em_tree = &inode->extent_tree;
+ struct extent_io_tree *io_tree = &inode->io_tree;
struct btrfs_trans_handle *trans = NULL;
const bool new_inline = !page || create;
goto not_found_em;
}
- btrfs_extent_item_to_extent_map(inode, path, item, new_inline, em);
+ btrfs_extent_item_to_extent_map(inode, path, item,
+ new_inline, em);
if (found_type == BTRFS_FILE_EXTENT_REG ||
found_type == BTRFS_FILE_EXTENT_PREALLOC) {
write_unlock(&em_tree->lock);
out:
- trace_btrfs_get_extent(root, BTRFS_I(inode), em);
+ trace_btrfs_get_extent(root, inode, em);
btrfs_free_path(path);
if (trans) {
return em;
}
-struct extent_map *btrfs_get_extent_fiemap(struct inode *inode, struct page *page,
- size_t pg_offset, u64 start, u64 len,
- int create)
+struct extent_map *btrfs_get_extent_fiemap(struct btrfs_inode *inode,
+ struct page *page,
+ size_t pg_offset, u64 start, u64 len,
+ int create)
{
struct extent_map *em;
struct extent_map *hole_em = NULL;
em = NULL;
/* ok, we didn't find anything, lets look for delalloc */
- found = count_range_bits(&BTRFS_I(inode)->io_tree, &range_start,
+ found = count_range_bits(&inode->io_tree, &range_start,
end, len, EXTENT_DELALLOC, 1);
found_end = range_start + found;
if (found_end < range_start)
if (ret) {
if (em) {
free_extent_map(em);
- btrfs_drop_extent_cache(inode, start,
+ btrfs_drop_extent_cache(BTRFS_I(inode), start,
start + len - 1, 0);
}
em = ERR_PTR(ret);
* doing DIO to, so we need to make sure there's no ordered
* extents in this range.
*/
- ordered = btrfs_lookup_ordered_range(inode, lockstart,
+ ordered = btrfs_lookup_ordered_range(BTRFS_I(inode), lockstart,
lockend - lockstart + 1);
/*
}
do {
- btrfs_drop_extent_cache(inode, em->start,
+ btrfs_drop_extent_cache(BTRFS_I(inode), em->start,
em->start + em->len - 1, 0);
write_lock(&em_tree->lock);
ret = add_extent_mapping(em_tree, em, 1);
goto err;
}
- em = btrfs_get_extent(inode, NULL, 0, start, len, 0);
+ em = btrfs_get_extent(BTRFS_I(inode), NULL, 0, start, len, 0);
if (IS_ERR(em)) {
ret = PTR_ERR(em);
goto unlock_err;
ret = btrfs_check_dio_repairable(inode, failed_bio, failrec,
failed_mirror);
if (!ret) {
- free_io_failure(inode, failrec);
+ free_io_failure(BTRFS_I(inode), failrec);
return -EIO;
}
bio = btrfs_create_repair_bio(inode, failed_bio, failrec, page,
pgoff, isector, repair_endio, repair_arg);
if (!bio) {
- free_io_failure(inode, failrec);
+ free_io_failure(BTRFS_I(inode), failrec);
return -EIO;
}
bio_set_op_attrs(bio, REQ_OP_READ, read_mode);
ret = submit_dio_repair_bio(inode, bio, failrec->this_mirror);
if (ret) {
- free_io_failure(inode, failrec);
+ free_io_failure(BTRFS_I(inode), failrec);
bio_put(bio);
}
done->uptodate = 1;
bio_for_each_segment_all(bvec, bio, i)
- clean_io_failure(done->inode, done->start, bvec->bv_page, 0);
+ clean_io_failure(BTRFS_I(done->inode), done->start, bvec->bv_page, 0);
end:
complete(&done->done);
bio_put(bio);
bvec->bv_page, bvec->bv_offset,
done->start, bvec->bv_len);
if (!ret)
- clean_io_failure(done->inode, done->start,
+ clean_io_failure(BTRFS_I(done->inode), done->start,
bvec->bv_page, bvec->bv_offset);
else
uptodate = 0;
lock_extent_bits(tree, page_start, page_end, &cached_state);
again:
start = page_start;
- ordered = btrfs_lookup_ordered_range(inode, start,
+ ordered = btrfs_lookup_ordered_range(BTRFS_I(inode), start,
page_end - start + 1);
if (ordered) {
end = min(page_end, ordered->file_offset + ordered->len - 1);
* we can't set the delalloc bits if there are pending ordered
* extents. Drop our locks and wait for them to finish
*/
- ordered = btrfs_lookup_ordered_range(inode, page_start, PAGE_SIZE);
+ ordered = btrfs_lookup_ordered_range(BTRFS_I(inode), page_start,
+ PAGE_SIZE);
if (ordered) {
unlock_extent_cached(io_tree, page_start, page_end,
&cached_state, GFP_NOFS);
if (ret == 0 && inode->i_nlink > 0) {
trans->block_rsv = root->orphan_block_rsv;
- ret = btrfs_orphan_del(trans, inode);
+ ret = btrfs_orphan_del(trans, BTRFS_I(inode));
if (ret)
err = ret;
}
inode->i_fop = &btrfs_dir_file_operations;
set_nlink(inode, 1);
- btrfs_i_size_write(inode, 0);
+ btrfs_i_size_write(BTRFS_I(inode), 0);
unlock_new_inode(inode);
err = btrfs_subvol_inherit_props(trans, new_root, parent_root);
#ifdef CONFIG_BTRFS_FS_RUN_SANITY_TESTS
void btrfs_test_destroy_inode(struct inode *inode)
{
- btrfs_drop_extent_cache(inode, 0, (u64)-1, 0);
+ btrfs_drop_extent_cache(BTRFS_I(inode), 0, (u64)-1, 0);
kmem_cache_free(btrfs_inode_cachep, BTRFS_I(inode));
}
#endif
}
btrfs_qgroup_check_reserved_leak(inode);
inode_tree_del(inode);
- btrfs_drop_extent_cache(inode, 0, (u64)-1, 0);
+ btrfs_drop_extent_cache(BTRFS_I(inode), 0, (u64)-1, 0);
free:
call_rcu(&inode->i_rcu, btrfs_i_callback);
}
return -ENOMEM;
}
-static int btrfs_getattr(struct vfsmount *mnt,
- struct dentry *dentry, struct kstat *stat)
+static int btrfs_getattr(const struct path *path, struct kstat *stat,
+ u32 request_mask, unsigned int flags)
{
u64 delalloc_bytes;
- struct inode *inode = d_inode(dentry);
+ struct inode *inode = d_inode(path->dentry);
u32 blocksize = inode->i_sb->s_blocksize;
generic_fillattr(inode, stat);
* We need to find a free sequence number both in the source and
* in the destination directory for the exchange.
*/
- ret = btrfs_set_inode_index(new_dir, &old_idx);
+ ret = btrfs_set_inode_index(BTRFS_I(new_dir), &old_idx);
if (ret)
goto out_fail;
- ret = btrfs_set_inode_index(old_dir, &new_idx);
+ ret = btrfs_set_inode_index(BTRFS_I(old_dir), &new_idx);
if (ret)
goto out_fail;
goto out_fail;
}
- ret = btrfs_add_link(trans, new_dir, old_inode,
+ ret = btrfs_add_link(trans, BTRFS_I(new_dir), BTRFS_I(old_inode),
new_dentry->d_name.name,
new_dentry->d_name.len, 0, old_idx);
if (ret) {
goto out_fail;
}
- ret = btrfs_add_link(trans, old_dir, new_inode,
+ ret = btrfs_add_link(trans, BTRFS_I(old_dir), BTRFS_I(new_inode),
old_dentry->d_name.name,
old_dentry->d_name.len, 0, new_idx);
if (ret) {
if (ret)
goto out;
- ret = btrfs_add_nondir(trans, dir, dentry,
- inode, 0, index);
+ ret = btrfs_add_nondir(trans, BTRFS_I(dir), dentry,
+ BTRFS_I(inode), 0, index);
if (ret)
goto out;
if (dest != root)
btrfs_record_root_in_trans(trans, dest);
- ret = btrfs_set_inode_index(new_dir, &index);
+ ret = btrfs_set_inode_index(BTRFS_I(new_dir), &index);
if (ret)
goto out_fail;
new_dentry->d_name.len);
}
if (!ret && new_inode->i_nlink == 0)
- ret = btrfs_orphan_add(trans, d_inode(new_dentry));
+ ret = btrfs_orphan_add(trans,
+ BTRFS_I(d_inode(new_dentry)));
if (ret) {
btrfs_abort_transaction(trans, ret);
goto out_fail;
}
}
- ret = btrfs_add_link(trans, new_dir, old_inode,
+ ret = btrfs_add_link(trans, BTRFS_I(new_dir), BTRFS_I(old_inode),
new_dentry->d_name.name,
new_dentry->d_name.len, 0, index);
if (ret) {
inode_nohighmem(inode);
inode->i_mapping->a_ops = &btrfs_symlink_aops;
inode_set_bytes(inode, name_len);
- btrfs_i_size_write(inode, name_len);
+ btrfs_i_size_write(BTRFS_I(inode), name_len);
err = btrfs_update_inode(trans, root, inode);
/*
* Last step, add directory indexes for our symlink inode. This is the
* elsewhere above.
*/
if (!err)
- err = btrfs_add_nondir(trans, dir, dentry, inode, 0, index);
+ err = btrfs_add_nondir(trans, BTRFS_I(dir), dentry,
+ BTRFS_I(inode), 0, index);
if (err) {
drop_inode = 1;
goto out_unlock_inode;
break;
}
- btrfs_drop_extent_cache(inode, cur_offset,
+ btrfs_drop_extent_cache(BTRFS_I(inode), cur_offset,
cur_offset + ins.offset -1, 0);
em = alloc_extent_map();
write_unlock(&em_tree->lock);
if (ret != -EEXIST)
break;
- btrfs_drop_extent_cache(inode, cur_offset,
+ btrfs_drop_extent_cache(BTRFS_I(inode), cur_offset,
cur_offset + ins.offset - 1,
0);
}
ret = btrfs_update_inode(trans, root, inode);
if (ret)
goto out_inode;
- ret = btrfs_orphan_add(trans, inode);
+ ret = btrfs_orphan_add(trans, BTRFS_I(inode));
if (ret)
goto out_inode;
}
+__attribute__((const))
+static int dummy_readpage_io_failed_hook(struct page *page, int failed_mirror)
+{
+ return 0;
+}
+
static const struct inode_operations btrfs_dir_inode_operations = {
.getattr = btrfs_getattr,
.lookup = btrfs_lookup,
};
static const struct extent_io_ops btrfs_extent_io_ops = {
- .fill_delalloc = run_delalloc_range,
+ /* mandatory callbacks */
.submit_bio_hook = btrfs_submit_bio_hook,
- .merge_bio_hook = btrfs_merge_bio_hook,
.readpage_end_io_hook = btrfs_readpage_end_io_hook,
+ .merge_bio_hook = btrfs_merge_bio_hook,
+ .readpage_io_failed_hook = dummy_readpage_io_failed_hook,
+
+ /* optional callbacks */
+ .fill_delalloc = run_delalloc_range,
.writepage_end_io_hook = btrfs_writepage_end_io_hook,
.writepage_start_hook = btrfs_writepage_start_hook,
.set_bit_hook = btrfs_set_bit_hook,
static noinline int create_subvol(struct inode *dir,
struct dentry *dentry,
- char *name, int namelen,
+ const char *name, int namelen,
u64 *async_transid,
struct btrfs_qgroup_inherit *inherit)
{
/*
* insert the directory item
*/
- ret = btrfs_set_inode_index(dir, &index);
+ ret = btrfs_set_inode_index(BTRFS_I(dir), &index);
if (ret) {
btrfs_abort_transaction(trans, ret);
goto fail;
}
ret = btrfs_insert_dir_item(trans, root,
- name, namelen, dir, &key,
+ name, namelen, BTRFS_I(dir), &key,
BTRFS_FT_DIR, index);
if (ret) {
btrfs_abort_transaction(trans, ret);
goto fail;
}
- btrfs_i_size_write(dir, dir->i_size + namelen * 2);
+ btrfs_i_size_write(BTRFS_I(dir), dir->i_size + namelen * 2);
ret = btrfs_update_inode(trans, root, dir);
BUG_ON(ret);
* inside this filesystem so it's quite a bit simpler.
*/
static noinline int btrfs_mksubvol(const struct path *parent,
- char *name, int namelen,
+ const char *name, int namelen,
struct btrfs_root *snap_src,
u64 *async_transid, bool readonly,
struct btrfs_qgroup_inherit *inherit)
/* get the big lock and read metadata off disk */
lock_extent_bits(io_tree, start, end, &cached);
- em = btrfs_get_extent(inode, NULL, 0, start, len, 0);
+ em = btrfs_get_extent(BTRFS_I(inode), NULL, 0, start, len, 0);
unlock_extent_cached(io_tree, start, end, &cached, GFP_NOFS);
if (IS_ERR(em))
}
static noinline int btrfs_ioctl_snap_create_transid(struct file *file,
- char *name, unsigned long fd, int subvol,
+ const char *name, unsigned long fd, int subvol,
u64 *transid, bool readonly,
struct btrfs_qgroup_inherit *inherit)
{
if (endoff > destoff + olen)
endoff = destoff + olen;
if (endoff > inode->i_size)
- btrfs_i_size_write(inode, endoff);
+ btrfs_i_size_write(BTRFS_I(inode), endoff);
ret = btrfs_update_inode(trans, root, inode);
if (ret) {
return ret;
}
-static void clone_update_extent_map(struct inode *inode,
+static void clone_update_extent_map(struct btrfs_inode *inode,
const struct btrfs_trans_handle *trans,
const struct btrfs_path *path,
const u64 hole_offset,
const u64 hole_len)
{
- struct extent_map_tree *em_tree = &BTRFS_I(inode)->extent_tree;
+ struct extent_map_tree *em_tree = &inode->extent_tree;
struct extent_map *em;
int ret;
em = alloc_extent_map();
if (!em) {
- set_bit(BTRFS_INODE_NEEDS_FULL_SYNC,
- &BTRFS_I(inode)->runtime_flags);
+ set_bit(BTRFS_INODE_NEEDS_FULL_SYNC, &inode->runtime_flags);
return;
}
if (btrfs_file_extent_type(path->nodes[0], fi) ==
BTRFS_FILE_EXTENT_INLINE)
set_bit(BTRFS_INODE_NEEDS_FULL_SYNC,
- &BTRFS_I(inode)->runtime_flags);
+ &inode->runtime_flags);
} else {
em->start = hole_offset;
em->len = hole_len;
}
if (ret)
- set_bit(BTRFS_INODE_NEEDS_FULL_SYNC,
- &BTRFS_I(inode)->runtime_flags);
+ set_bit(BTRFS_INODE_NEEDS_FULL_SYNC, &inode->runtime_flags);
}
/*
/* If we have an implicit hole (NO_HOLES feature). */
if (drop_start < new_key.offset)
- clone_update_extent_map(inode, trans,
+ clone_update_extent_map(BTRFS_I(inode), trans,
NULL, drop_start,
new_key.offset - drop_start);
- clone_update_extent_map(inode, trans, path, 0, 0);
+ clone_update_extent_map(BTRFS_I(inode), trans,
+ path, 0, 0);
btrfs_mark_buffer_dirty(leaf);
btrfs_release_path(path);
btrfs_end_transaction(trans);
goto out;
}
- clone_update_extent_map(inode, trans, NULL, last_dest_end,
- destoff + len - last_dest_end);
+ clone_update_extent_map(BTRFS_I(inode), trans, NULL,
+ last_dest_end,
+ destoff + len - last_dest_end);
ret = clone_finish_inode_update(trans, inode, destoff + len,
destoff, olen, no_time_update);
}
memcpy(buf, &dlen, LZO_LEN);
}
-static inline size_t read_compress_length(char *buf)
+static inline size_t read_compress_length(const char *buf)
{
__le32 dlen;
static int lzo_compress_pages(struct list_head *ws,
struct address_space *mapping,
- u64 start, unsigned long len,
+ u64 start,
struct page **pages,
- unsigned long nr_dest_pages,
unsigned long *out_pages,
unsigned long *total_in,
- unsigned long *total_out,
- unsigned long max_out)
+ unsigned long *total_out)
{
struct workspace *workspace = list_entry(ws, struct workspace, list);
int ret = 0;
struct page *in_page = NULL;
struct page *out_page = NULL;
unsigned long bytes_left;
-
+ unsigned long len = *total_out;
+ unsigned long nr_dest_pages = *out_pages;
+ const unsigned long max_out = nr_dest_pages * PAGE_SIZE;
size_t in_len;
size_t out_len;
char *buf;
/* Since the DIO code tries to lock a wide area we need to look for any ordered
* extents that exist in the range, rather than just the start of the range.
*/
-struct btrfs_ordered_extent *btrfs_lookup_ordered_range(struct inode *inode,
- u64 file_offset,
- u64 len)
+struct btrfs_ordered_extent *btrfs_lookup_ordered_range(
+ struct btrfs_inode *inode, u64 file_offset, u64 len)
{
struct btrfs_ordered_inode_tree *tree;
struct rb_node *node;
struct btrfs_ordered_extent *entry = NULL;
- tree = &BTRFS_I(inode)->ordered_tree;
+ tree = &inode->ordered_tree;
spin_lock_irq(&tree->lock);
node = tree_search(tree, file_offset);
if (!node) {
{
struct btrfs_ordered_extent *oe;
- oe = btrfs_lookup_ordered_range(inode, file_offset, len);
+ oe = btrfs_lookup_ordered_range(BTRFS_I(inode), file_offset, len);
if (oe) {
btrfs_put_ordered_extent(oe);
return true;
int btrfs_wait_ordered_range(struct inode *inode, u64 start, u64 len);
struct btrfs_ordered_extent *
btrfs_lookup_first_ordered_extent(struct inode * inode, u64 file_offset);
-struct btrfs_ordered_extent *btrfs_lookup_ordered_range(struct inode *inode,
- u64 file_offset,
- u64 len);
+struct btrfs_ordered_extent *btrfs_lookup_ordered_range(
+ struct btrfs_inode *inode,
+ u64 file_offset,
+ u64 len);
bool btrfs_have_ordered_extents_in_range(struct inode *inode,
u64 file_offset,
u64 len);
if (!ret)
continue;
- btrfs_drop_extent_cache(inode, key.offset, end,
- 1);
+ btrfs_drop_extent_cache(BTRFS_I(inode),
+ key.offset, end, 1);
unlock_extent(&BTRFS_I(inode)->io_tree,
key.offset, end);
}
/* the lock_extent waits for readpage to complete */
lock_extent(&BTRFS_I(inode)->io_tree, start, end);
- btrfs_drop_extent_cache(inode, start, end, 1);
+ btrfs_drop_extent_cache(BTRFS_I(inode), start, end, 1);
unlock_extent(&BTRFS_I(inode)->io_tree, start, end);
}
return 0;
free_extent_map(em);
break;
}
- btrfs_drop_extent_cache(inode, start, end, 0);
+ btrfs_drop_extent_cache(BTRFS_I(inode), start, end, 0);
}
unlock_extent(&BTRFS_I(inode)->io_tree, start, end);
return ret;
index = (cluster->start - offset) >> PAGE_SHIFT;
last_index = (cluster->end - offset) >> PAGE_SHIFT;
while (index <= last_index) {
- ret = btrfs_delalloc_reserve_metadata(inode, PAGE_SIZE);
+ ret = btrfs_delalloc_reserve_metadata(BTRFS_I(inode),
+ PAGE_SIZE);
if (ret)
goto out;
page = find_or_create_page(inode->i_mapping, index,
mask);
if (!page) {
- btrfs_delalloc_release_metadata(inode,
+ btrfs_delalloc_release_metadata(BTRFS_I(inode),
PAGE_SIZE);
ret = -ENOMEM;
goto out;
if (!PageUptodate(page)) {
unlock_page(page);
put_page(page);
- btrfs_delalloc_release_metadata(inode,
+ btrfs_delalloc_release_metadata(BTRFS_I(inode),
PAGE_SIZE);
ret = -EIO;
goto out;
BUG_ON(IS_ERR(inode) || is_bad_inode(inode));
BTRFS_I(inode)->index_cnt = group->key.objectid;
- err = btrfs_orphan_add(trans, inode);
+ err = btrfs_orphan_add(trans, BTRFS_I(inode));
out:
btrfs_end_transaction(trans);
btrfs_btree_balance_dirty(fs_info);
ret = -EIO;
goto out;
}
- ret = repair_io_failure(inode, offset, PAGE_SIZE,
+ ret = repair_io_failure(BTRFS_I(inode), offset, PAGE_SIZE,
fixup->logical, page,
offset - page_offset(page),
fixup->mirror_num);
scrub_pending_trans_workers_dec(sctx);
}
-static int check_extent_to_block(struct inode *inode, u64 start, u64 len,
+static int check_extent_to_block(struct btrfs_inode *inode, u64 start, u64 len,
u64 logical)
{
struct extent_state *cached_state = NULL;
u64 lockstart = start, lockend = start + len - 1;
int ret = 0;
- io_tree = &BTRFS_I(inode)->io_tree;
+ io_tree = &inode->io_tree;
lock_extent_bits(io_tree, lockstart, lockend, &cached_state);
ordered = btrfs_lookup_ordered_range(inode, lockstart, len);
io_tree = &BTRFS_I(inode)->io_tree;
nocow_ctx_logical = nocow_ctx->logical;
- ret = check_extent_to_block(inode, offset, len, nocow_ctx_logical);
+ ret = check_extent_to_block(BTRFS_I(inode), offset, len,
+ nocow_ctx_logical);
if (ret) {
ret = ret > 0 ? 0 : ret;
goto out;
}
}
- ret = check_extent_to_block(inode, offset, len,
+ ret = check_extent_to_block(BTRFS_I(inode), offset, len,
nocow_ctx_logical);
if (ret) {
ret = ret > 0 ? 0 : ret;
{
int ret;
+ if (ino == BTRFS_FIRST_FREE_OBJECTID)
+ return 1;
+
ret = get_cur_inode_state(sctx, ino, gen);
if (ret < 0)
goto out;
* not deleted and then re-created, if it was then we have no overwrite
* and we can just unlink this entry.
*/
- if (sctx->parent_root) {
+ if (sctx->parent_root && dir != BTRFS_FIRST_FREE_OBJECTID) {
ret = get_inode_info(sctx->parent_root, dir, NULL, &gen, NULL,
NULL, NULL, NULL);
if (ret < 0 && ret != -ENOENT)
if (ret <= 0)
goto out;
+ if (dir != BTRFS_FIRST_FREE_OBJECTID) {
+ ret = get_inode_info(sctx->send_root, dir, NULL, &gen, NULL,
+ NULL, NULL, NULL);
+ if (ret < 0 && ret != -ENOENT)
+ goto out;
+ if (ret) {
+ ret = 0;
+ goto out;
+ }
+ if (gen != dir_gen)
+ goto out;
+ }
+
/* check if the ref was overwritten by another ref */
ret = lookup_dir_item_inode(sctx->send_root, dir, name, name_len,
&ow_inode, &other_type);
{
int ret = 0;
u64 ino = parent_ref->dir;
+ u64 ino_gen = parent_ref->dir_gen;
u64 parent_ino_before, parent_ino_after;
struct fs_path *path_before = NULL;
struct fs_path *path_after = NULL;
* at get_cur_path()).
*/
while (ino > BTRFS_FIRST_FREE_OBJECTID) {
+ u64 parent_ino_after_gen;
+
if (is_waiting_for_move(sctx, ino)) {
/*
* If the current inode is an ancestor of ino in the
fs_path_reset(path_after);
ret = get_first_ref(sctx->send_root, ino, &parent_ino_after,
- NULL, path_after);
+ &parent_ino_after_gen, path_after);
if (ret < 0)
goto out;
ret = get_first_ref(sctx->parent_root, ino, &parent_ino_before,
if (ino > sctx->cur_ino &&
(parent_ino_before != parent_ino_after || len1 != len2 ||
memcmp(path_before->start, path_after->start, len1))) {
- ret = 1;
- break;
+ u64 parent_ino_gen;
+
+ ret = get_inode_info(sctx->parent_root, ino, NULL,
+ &parent_ino_gen, NULL, NULL, NULL,
+ NULL);
+ if (ret < 0)
+ goto out;
+ if (ino_gen == parent_ino_gen) {
+ ret = 1;
+ break;
+ }
}
ino = parent_ino_after;
+ ino_gen = parent_ino_after_gen;
}
out:
return ret;
}
+static int range_is_hole_in_parent(struct send_ctx *sctx,
+ const u64 start,
+ const u64 end)
+{
+ struct btrfs_path *path;
+ struct btrfs_key key;
+ struct btrfs_root *root = sctx->parent_root;
+ u64 search_start = start;
+ int ret;
+
+ path = alloc_path_for_send();
+ if (!path)
+ return -ENOMEM;
+
+ key.objectid = sctx->cur_ino;
+ key.type = BTRFS_EXTENT_DATA_KEY;
+ key.offset = search_start;
+ ret = btrfs_search_slot(NULL, root, &key, path, 0, 0);
+ if (ret < 0)
+ goto out;
+ if (ret > 0 && path->slots[0] > 0)
+ path->slots[0]--;
+
+ while (search_start < end) {
+ struct extent_buffer *leaf = path->nodes[0];
+ int slot = path->slots[0];
+ struct btrfs_file_extent_item *fi;
+ u64 extent_end;
+
+ if (slot >= btrfs_header_nritems(leaf)) {
+ ret = btrfs_next_leaf(root, path);
+ if (ret < 0)
+ goto out;
+ else if (ret > 0)
+ break;
+ continue;
+ }
+
+ btrfs_item_key_to_cpu(leaf, &key, slot);
+ if (key.objectid < sctx->cur_ino ||
+ key.type < BTRFS_EXTENT_DATA_KEY)
+ goto next;
+ if (key.objectid > sctx->cur_ino ||
+ key.type > BTRFS_EXTENT_DATA_KEY ||
+ key.offset >= end)
+ break;
+
+ fi = btrfs_item_ptr(leaf, slot, struct btrfs_file_extent_item);
+ if (btrfs_file_extent_type(leaf, fi) ==
+ BTRFS_FILE_EXTENT_INLINE) {
+ u64 size = btrfs_file_extent_inline_len(leaf, slot, fi);
+
+ extent_end = ALIGN(key.offset + size,
+ root->fs_info->sectorsize);
+ } else {
+ extent_end = key.offset +
+ btrfs_file_extent_num_bytes(leaf, fi);
+ }
+ if (extent_end <= start)
+ goto next;
+ if (btrfs_file_extent_disk_bytenr(leaf, fi) == 0) {
+ search_start = extent_end;
+ goto next;
+ }
+ ret = 0;
+ goto out;
+next:
+ path->slots[0]++;
+ }
+ ret = 1;
+out:
+ btrfs_free_path(path);
+ return ret;
+}
+
static int maybe_send_hole(struct send_ctx *sctx, struct btrfs_path *path,
struct btrfs_key *key)
{
return ret;
}
- if (sctx->cur_inode_last_extent < key->offset)
- ret = send_hole(sctx, key->offset);
+ if (sctx->cur_inode_last_extent < key->offset) {
+ ret = range_is_hole_in_parent(sctx,
+ sctx->cur_inode_last_extent,
+ key->offset);
+ if (ret < 0)
+ return ret;
+ else if (ret == 0)
+ ret = send_hole(sctx, key->offset);
+ else
+ ret = 0;
+ }
sctx->cur_inode_last_extent = extent_end;
return ret;
}
/* First with no extents */
BTRFS_I(inode)->root = root;
- em = btrfs_get_extent(inode, NULL, 0, 0, sectorsize, 0);
+ em = btrfs_get_extent(BTRFS_I(inode), NULL, 0, 0, sectorsize, 0);
if (IS_ERR(em)) {
em = NULL;
test_msg("Got an error when we shouldn't have\n");
goto out;
}
free_extent_map(em);
- btrfs_drop_extent_cache(inode, 0, (u64)-1, 0);
+ btrfs_drop_extent_cache(BTRFS_I(inode), 0, (u64)-1, 0);
/*
* All of the magic numbers are based on the mapping setup in
*/
setup_file_extents(root, sectorsize);
- em = btrfs_get_extent(inode, NULL, 0, 0, (u64)-1, 0);
+ em = btrfs_get_extent(BTRFS_I(inode), NULL, 0, 0, (u64)-1, 0);
if (IS_ERR(em)) {
test_msg("Got an error when we shouldn't have\n");
goto out;
offset = em->start + em->len;
free_extent_map(em);
- em = btrfs_get_extent(inode, NULL, 0, offset, sectorsize, 0);
+ em = btrfs_get_extent(BTRFS_I(inode), NULL, 0, offset, sectorsize, 0);
if (IS_ERR(em)) {
test_msg("Got an error when we shouldn't have\n");
goto out;
offset = em->start + em->len;
free_extent_map(em);
- em = btrfs_get_extent(inode, NULL, 0, offset, sectorsize, 0);
+ em = btrfs_get_extent(BTRFS_I(inode), NULL, 0, offset, sectorsize, 0);
if (IS_ERR(em)) {
test_msg("Got an error when we shouldn't have\n");
goto out;
free_extent_map(em);
/* Regular extent */
- em = btrfs_get_extent(inode, NULL, 0, offset, sectorsize, 0);
+ em = btrfs_get_extent(BTRFS_I(inode), NULL, 0, offset, sectorsize, 0);
if (IS_ERR(em)) {
test_msg("Got an error when we shouldn't have\n");
goto out;
free_extent_map(em);
/* The next 3 are split extents */
- em = btrfs_get_extent(inode, NULL, 0, offset, sectorsize, 0);
+ em = btrfs_get_extent(BTRFS_I(inode), NULL, 0, offset, sectorsize, 0);
if (IS_ERR(em)) {
test_msg("Got an error when we shouldn't have\n");
goto out;
offset = em->start + em->len;
free_extent_map(em);
- em = btrfs_get_extent(inode, NULL, 0, offset, sectorsize, 0);
+ em = btrfs_get_extent(BTRFS_I(inode), NULL, 0, offset, sectorsize, 0);
if (IS_ERR(em)) {
test_msg("Got an error when we shouldn't have\n");
goto out;
offset = em->start + em->len;
free_extent_map(em);
- em = btrfs_get_extent(inode, NULL, 0, offset, sectorsize, 0);
+ em = btrfs_get_extent(BTRFS_I(inode), NULL, 0, offset, sectorsize, 0);
if (IS_ERR(em)) {
test_msg("Got an error when we shouldn't have\n");
goto out;
free_extent_map(em);
/* Prealloc extent */
- em = btrfs_get_extent(inode, NULL, 0, offset, sectorsize, 0);
+ em = btrfs_get_extent(BTRFS_I(inode), NULL, 0, offset, sectorsize, 0);
if (IS_ERR(em)) {
test_msg("Got an error when we shouldn't have\n");
goto out;
free_extent_map(em);
/* The next 3 are a half written prealloc extent */
- em = btrfs_get_extent(inode, NULL, 0, offset, sectorsize, 0);
+ em = btrfs_get_extent(BTRFS_I(inode), NULL, 0, offset, sectorsize, 0);
if (IS_ERR(em)) {
test_msg("Got an error when we shouldn't have\n");
goto out;
offset = em->start + em->len;
free_extent_map(em);
- em = btrfs_get_extent(inode, NULL, 0, offset, sectorsize, 0);
+ em = btrfs_get_extent(BTRFS_I(inode), NULL, 0, offset, sectorsize, 0);
if (IS_ERR(em)) {
test_msg("Got an error when we shouldn't have\n");
goto out;
offset = em->start + em->len;
free_extent_map(em);
- em = btrfs_get_extent(inode, NULL, 0, offset, sectorsize, 0);
+ em = btrfs_get_extent(BTRFS_I(inode), NULL, 0, offset, sectorsize, 0);
if (IS_ERR(em)) {
test_msg("Got an error when we shouldn't have\n");
goto out;
free_extent_map(em);
/* Now for the compressed extent */
- em = btrfs_get_extent(inode, NULL, 0, offset, sectorsize, 0);
+ em = btrfs_get_extent(BTRFS_I(inode), NULL, 0, offset, sectorsize, 0);
if (IS_ERR(em)) {
test_msg("Got an error when we shouldn't have\n");
goto out;
free_extent_map(em);
/* Split compressed extent */
- em = btrfs_get_extent(inode, NULL, 0, offset, sectorsize, 0);
+ em = btrfs_get_extent(BTRFS_I(inode), NULL, 0, offset, sectorsize, 0);
if (IS_ERR(em)) {
test_msg("Got an error when we shouldn't have\n");
goto out;
offset = em->start + em->len;
free_extent_map(em);
- em = btrfs_get_extent(inode, NULL, 0, offset, sectorsize, 0);
+ em = btrfs_get_extent(BTRFS_I(inode), NULL, 0, offset, sectorsize, 0);
if (IS_ERR(em)) {
test_msg("Got an error when we shouldn't have\n");
goto out;
offset = em->start + em->len;
free_extent_map(em);
- em = btrfs_get_extent(inode, NULL, 0, offset, sectorsize, 0);
+ em = btrfs_get_extent(BTRFS_I(inode), NULL, 0, offset, sectorsize, 0);
if (IS_ERR(em)) {
test_msg("Got an error when we shouldn't have\n");
goto out;
free_extent_map(em);
/* A hole between regular extents but no hole extent */
- em = btrfs_get_extent(inode, NULL, 0, offset + 6, sectorsize, 0);
+ em = btrfs_get_extent(BTRFS_I(inode), NULL, 0, offset + 6,
+ sectorsize, 0);
if (IS_ERR(em)) {
test_msg("Got an error when we shouldn't have\n");
goto out;
offset = em->start + em->len;
free_extent_map(em);
- em = btrfs_get_extent(inode, NULL, 0, offset, 4096 * 1024, 0);
+ em = btrfs_get_extent(BTRFS_I(inode), NULL, 0, offset, 4096 * 1024, 0);
if (IS_ERR(em)) {
test_msg("Got an error when we shouldn't have\n");
goto out;
offset = em->start + em->len;
free_extent_map(em);
- em = btrfs_get_extent(inode, NULL, 0, offset, sectorsize, 0);
+ em = btrfs_get_extent(BTRFS_I(inode), NULL, 0, offset, sectorsize, 0);
if (IS_ERR(em)) {
test_msg("Got an error when we shouldn't have\n");
goto out;
insert_inode_item_key(root);
insert_extent(root, sectorsize, sectorsize, sectorsize, 0, sectorsize,
sectorsize, BTRFS_FILE_EXTENT_REG, 0, 1);
- em = btrfs_get_extent(inode, NULL, 0, 0, 2 * sectorsize, 0);
+ em = btrfs_get_extent(BTRFS_I(inode), NULL, 0, 0, 2 * sectorsize, 0);
if (IS_ERR(em)) {
test_msg("Got an error when we shouldn't have\n");
goto out;
}
free_extent_map(em);
- em = btrfs_get_extent(inode, NULL, 0, sectorsize, 2 * sectorsize, 0);
+ em = btrfs_get_extent(BTRFS_I(inode), NULL, 0, sectorsize,
+ 2 * sectorsize, 0);
if (IS_ERR(em)) {
test_msg("Got an error when we shouldn't have\n");
goto out;
/*
* insert the directory item
*/
- ret = btrfs_set_inode_index(parent_inode, &index);
+ ret = btrfs_set_inode_index(BTRFS_I(parent_inode), &index);
BUG_ON(ret); /* -ENOMEM */
/* check if there is a file/dir which has the same name. */
ret = btrfs_insert_dir_item(trans, parent_root,
dentry->d_name.name, dentry->d_name.len,
- parent_inode, &key,
+ BTRFS_I(parent_inode), &key,
BTRFS_FT_DIR, index);
/* We have check then name at the beginning, so it is impossible. */
BUG_ON(ret == -EEXIST || ret == -EOVERFLOW);
goto fail;
}
- btrfs_i_size_write(parent_inode, parent_inode->i_size +
+ btrfs_i_size_write(BTRFS_I(parent_inode), parent_inode->i_size +
dentry->d_name.len * 2);
parent_inode->i_mtime = parent_inode->i_ctime =
current_time(parent_inode);
unsigned long dest_offset;
struct btrfs_key ins;
+ if (btrfs_file_extent_disk_bytenr(eb, item) == 0 &&
+ btrfs_fs_incompat(fs_info, NO_HOLES))
+ goto update_inode;
+
ret = btrfs_insert_empty_item(trans, root, path, key,
sizeof(*item));
if (ret)
}
inode_add_bytes(inode, nbytes);
+update_inode:
ret = btrfs_update_inode(trans, root, inode);
out:
if (inode)
}
/* insert our name */
- ret = btrfs_add_link(trans, dir, inode, name, namelen,
- 0, ref_index);
+ ret = btrfs_add_link(trans, BTRFS_I(dir),
+ BTRFS_I(inode),
+ name, namelen, 0, ref_index);
if (ret)
goto out;
return -EIO;
}
- ret = btrfs_add_link(trans, dir, inode, name, name_len, 1, index);
+ ret = btrfs_add_link(trans, BTRFS_I(dir), BTRFS_I(inode), name,
+ name_len, 1, index);
/* FIXME, put inode into FIXUP list */
out:
btrfs_release_path(path);
if (!ret && update_size) {
- btrfs_i_size_write(dir, dir->i_size + name_len * 2);
+ btrfs_i_size_write(BTRFS_I(dir), dir->i_size + name_len * 2);
ret = btrfs_update_inode(trans, root, dir);
}
kfree(name);
* a full commit is required.
*/
static noinline int check_parent_dirs_for_sync(struct btrfs_trans_handle *trans,
- struct inode *inode,
+ struct btrfs_inode *inode,
struct dentry *parent,
struct super_block *sb,
u64 last_committed)
{
int ret = 0;
struct dentry *old_parent = NULL;
- struct inode *orig_inode = inode;
+ struct btrfs_inode *orig_inode = inode;
/*
* for regular files, if its inode is already on disk, we don't
* we can use the last_unlink_trans field to record renames
* and other fun in this file.
*/
- if (S_ISREG(inode->i_mode) &&
- BTRFS_I(inode)->generation <= last_committed &&
- BTRFS_I(inode)->last_unlink_trans <= last_committed)
- goto out;
+ if (S_ISREG(inode->vfs_inode.i_mode) &&
+ inode->generation <= last_committed &&
+ inode->last_unlink_trans <= last_committed)
+ goto out;
- if (!S_ISDIR(inode->i_mode)) {
+ if (!S_ISDIR(inode->vfs_inode.i_mode)) {
if (!parent || d_really_is_negative(parent) || sb != parent->d_sb)
goto out;
- inode = d_inode(parent);
+ inode = BTRFS_I(d_inode(parent));
}
while (1) {
* think this inode has already been logged.
*/
if (inode != orig_inode)
- BTRFS_I(inode)->logged_trans = trans->transid;
+ inode->logged_trans = trans->transid;
smp_mb();
- if (btrfs_must_commit_transaction(trans, BTRFS_I(inode))) {
+ if (btrfs_must_commit_transaction(trans, inode)) {
ret = 1;
break;
}
break;
if (IS_ROOT(parent)) {
- inode = d_inode(parent);
- if (btrfs_must_commit_transaction(trans, BTRFS_I(inode)))
+ inode = BTRFS_I(d_inode(parent));
+ if (btrfs_must_commit_transaction(trans, inode))
ret = 1;
break;
}
parent = dget_parent(parent);
dput(old_parent);
old_parent = parent;
- inode = d_inode(parent);
+ inode = BTRFS_I(d_inode(parent));
}
dput(old_parent);
}
static int btrfs_log_all_parents(struct btrfs_trans_handle *trans,
- struct inode *inode,
+ struct btrfs_inode *inode,
struct btrfs_log_ctx *ctx)
{
- struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
+ struct btrfs_fs_info *fs_info = btrfs_sb(inode->vfs_inode.i_sb);
int ret;
struct btrfs_path *path;
struct btrfs_key key;
- struct btrfs_root *root = BTRFS_I(inode)->root;
- const u64 ino = btrfs_ino(BTRFS_I(inode));
+ struct btrfs_root *root = inode->root;
+ const u64 ino = btrfs_ino(inode);
path = btrfs_alloc_path();
if (!path)
* the last committed transaction
*/
static int btrfs_log_inode_parent(struct btrfs_trans_handle *trans,
- struct btrfs_root *root, struct inode *inode,
+ struct btrfs_root *root,
+ struct btrfs_inode *inode,
struct dentry *parent,
const loff_t start,
const loff_t end,
int ret = 0;
u64 last_committed = fs_info->last_trans_committed;
bool log_dentries = false;
- struct inode *orig_inode = inode;
+ struct btrfs_inode *orig_inode = inode;
- sb = inode->i_sb;
+ sb = inode->vfs_inode.i_sb;
if (btrfs_test_opt(fs_info, NOTREELOG)) {
ret = 1;
goto end_no_trans;
}
- if (root != BTRFS_I(inode)->root ||
- btrfs_root_refs(&root->root_item) == 0) {
+ if (root != inode->root || btrfs_root_refs(&root->root_item) == 0) {
ret = 1;
goto end_no_trans;
}
- ret = check_parent_dirs_for_sync(trans, inode, parent,
- sb, last_committed);
+ ret = check_parent_dirs_for_sync(trans, inode, parent, sb,
+ last_committed);
if (ret)
goto end_no_trans;
- if (btrfs_inode_in_log(BTRFS_I(inode), trans->transid)) {
+ if (btrfs_inode_in_log(inode, trans->transid)) {
ret = BTRFS_NO_LOG_SYNC;
goto end_no_trans;
}
if (ret)
goto end_no_trans;
- ret = btrfs_log_inode(trans, root, BTRFS_I(inode), inode_only,
- start, end, ctx);
+ ret = btrfs_log_inode(trans, root, inode, inode_only, start, end, ctx);
if (ret)
goto end_trans;
* we can use the last_unlink_trans field to record renames
* and other fun in this file.
*/
- if (S_ISREG(inode->i_mode) &&
- BTRFS_I(inode)->generation <= last_committed &&
- BTRFS_I(inode)->last_unlink_trans <= last_committed) {
+ if (S_ISREG(inode->vfs_inode.i_mode) &&
+ inode->generation <= last_committed &&
+ inode->last_unlink_trans <= last_committed) {
ret = 0;
goto end_trans;
}
- if (S_ISDIR(inode->i_mode) && ctx && ctx->log_new_dentries)
+ if (S_ISDIR(inode->vfs_inode.i_mode) && ctx && ctx->log_new_dentries)
log_dentries = true;
/*
* but the file inode does not have a matching BTRFS_INODE_REF_KEY item
* and has a link count of 2.
*/
- if (BTRFS_I(inode)->last_unlink_trans > last_committed) {
+ if (inode->last_unlink_trans > last_committed) {
ret = btrfs_log_all_parents(trans, orig_inode, ctx);
if (ret)
goto end_trans;
if (!parent || d_really_is_negative(parent) || sb != parent->d_sb)
break;
- inode = d_inode(parent);
- if (root != BTRFS_I(inode)->root)
+ inode = BTRFS_I(d_inode(parent));
+ if (root != inode->root)
break;
- if (BTRFS_I(inode)->generation > last_committed) {
- ret = btrfs_log_inode(trans, root, BTRFS_I(inode),
- LOG_INODE_EXISTS,
- 0, LLONG_MAX, ctx);
+ if (inode->generation > last_committed) {
+ ret = btrfs_log_inode(trans, root, inode,
+ LOG_INODE_EXISTS, 0, LLONG_MAX, ctx);
if (ret)
goto end_trans;
}
old_parent = parent;
}
if (log_dentries)
- ret = log_new_dir_dentries(trans, root, BTRFS_I(orig_inode), ctx);
+ ret = log_new_dir_dentries(trans, root, orig_inode, ctx);
else
ret = 0;
end_trans:
struct dentry *parent = dget_parent(dentry);
int ret;
- ret = btrfs_log_inode_parent(trans, root, d_inode(dentry), parent,
- start, end, 0, ctx);
+ ret = btrfs_log_inode_parent(trans, root, BTRFS_I(d_inode(dentry)),
+ parent, start, end, 0, ctx);
dput(parent);
return ret;
(!old_dir || old_dir->logged_trans <= fs_info->last_trans_committed))
return 0;
- return btrfs_log_inode_parent(trans, root, &inode->vfs_inode, parent, 0,
+ return btrfs_log_inode_parent(trans, root, inode, parent, 0,
LLONG_MAX, 1, NULL);
}
* Function to update ctime/mtime for a given device path.
* Mainly used for ctime/mtime based probe like libblkid.
*/
-static void update_dev_time(char *path_name)
+static void update_dev_time(const char *path_name)
{
struct file *filp;
fs_info->fs_devices->latest_bdev = next_device->bdev;
}
-int btrfs_rm_device(struct btrfs_fs_info *fs_info, char *device_path, u64 devid)
+int btrfs_rm_device(struct btrfs_fs_info *fs_info, const char *device_path,
+ u64 devid)
{
struct btrfs_device *device;
struct btrfs_fs_devices *cur_devices;
}
static int btrfs_find_device_by_path(struct btrfs_fs_info *fs_info,
- char *device_path,
+ const char *device_path,
struct btrfs_device **device)
{
int ret = 0;
}
int btrfs_find_device_missing_or_by_path(struct btrfs_fs_info *fs_info,
- char *device_path,
+ const char *device_path,
struct btrfs_device **device)
{
*device = NULL;
* Lookup a device given by device id, or the path if the id is 0.
*/
int btrfs_find_device_by_devspec(struct btrfs_fs_info *fs_info, u64 devid,
- char *devpath, struct btrfs_device **device)
+ const char *devpath,
+ struct btrfs_device **device)
{
int ret;
return ret;
}
-int btrfs_init_new_device(struct btrfs_fs_info *fs_info, char *device_path)
+int btrfs_init_new_device(struct btrfs_fs_info *fs_info, const char *device_path)
{
struct btrfs_root *root = fs_info->dev_root;
struct request_queue *q;
}
int btrfs_init_dev_replace_tgtdev(struct btrfs_fs_info *fs_info,
- char *device_path,
+ const char *device_path,
struct btrfs_device *srcdev,
struct btrfs_device **device_out)
{
key.offset = device->devid;
path = btrfs_alloc_path();
- BUG_ON(!path);
+ if (!path)
+ return -ENOMEM;
ret = btrfs_search_slot(trans, dev_root, &key, path, -1, 1);
if (ret < 0) {
btrfs_warn_in_rcu(fs_info,
return 0;
}
-void btrfs_scratch_superblocks(struct block_device *bdev, char *device_path)
+void btrfs_scratch_superblocks(struct block_device *bdev, const char *device_path)
{
struct buffer_head *bh;
struct btrfs_super_block *disk_super;
void btrfs_assign_next_active_device(struct btrfs_fs_info *fs_info,
struct btrfs_device *device, struct btrfs_device *this_dev);
int btrfs_find_device_missing_or_by_path(struct btrfs_fs_info *fs_info,
- char *device_path,
+ const char *device_path,
struct btrfs_device **device);
int btrfs_find_device_by_devspec(struct btrfs_fs_info *fs_info, u64 devid,
- char *devpath,
+ const char *devpath,
struct btrfs_device **device);
struct btrfs_device *btrfs_alloc_device(struct btrfs_fs_info *fs_info,
const u64 *devid,
const u8 *uuid);
int btrfs_rm_device(struct btrfs_fs_info *fs_info,
- char *device_path, u64 devid);
+ const char *device_path, u64 devid);
void btrfs_cleanup_fs_uuids(void);
int btrfs_num_copies(struct btrfs_fs_info *fs_info, u64 logical, u64 len);
int btrfs_grow_device(struct btrfs_trans_handle *trans,
struct btrfs_device *btrfs_find_device(struct btrfs_fs_info *fs_info, u64 devid,
u8 *uuid, u8 *fsid);
int btrfs_shrink_device(struct btrfs_device *device, u64 new_size);
-int btrfs_init_new_device(struct btrfs_fs_info *fs_info, char *path);
+int btrfs_init_new_device(struct btrfs_fs_info *fs_info, const char *path);
int btrfs_init_dev_replace_tgtdev(struct btrfs_fs_info *fs_info,
- char *device_path,
+ const char *device_path,
struct btrfs_device *srcdev,
struct btrfs_device **device_out);
int btrfs_balance(struct btrfs_balance_control *bctl,
struct btrfs_device *tgtdev);
void btrfs_init_dev_replace_tgtdev_for_resume(struct btrfs_fs_info *fs_info,
struct btrfs_device *tgtdev);
-void btrfs_scratch_superblocks(struct block_device *bdev, char *device_path);
+void btrfs_scratch_superblocks(struct block_device *bdev, const char *device_path);
int btrfs_is_parity_mirror(struct btrfs_mapping_tree *map_tree,
u64 logical, u64 len, int mirror_num);
unsigned long btrfs_full_stripe_len(struct btrfs_fs_info *fs_info,
static int zlib_compress_pages(struct list_head *ws,
struct address_space *mapping,
- u64 start, unsigned long len,
+ u64 start,
struct page **pages,
- unsigned long nr_dest_pages,
unsigned long *out_pages,
unsigned long *total_in,
- unsigned long *total_out,
- unsigned long max_out)
+ unsigned long *total_out)
{
struct workspace *workspace = list_entry(ws, struct workspace, list);
int ret;
struct page *in_page = NULL;
struct page *out_page = NULL;
unsigned long bytes_left;
+ unsigned long len = *total_out;
+ unsigned long nr_dest_pages = *out_pages;
+ const unsigned long max_out = nr_dest_pages * PAGE_SIZE;
*out_pages = 0;
*total_out = 0;
*/
#include <linux/kernel.h>
+#include <linux/sched/signal.h>
#include <linux/syscalls.h>
#include <linux/fs.h>
#include <linux/iomap.h>
#include <linux/fscache-cache.h>
#include <linux/timer.h>
#include <linux/wait.h>
+#include <linux/cred.h>
#include <linux/workqueue.h>
#include <linux/security.h>
#include <linux/slab.h>
#include <linux/pagevec.h>
#include <linux/task_io_accounting_ops.h>
+#include <linux/signal.h>
#include "super.h"
#include "mds_client.h"
#include <linux/fs.h>
#include <linux/kernel.h>
-#include <linux/sched.h>
+#include <linux/sched/signal.h>
#include <linux/slab.h>
#include <linux/vmalloc.h>
#include <linux/wait.h>
* Get all attributes. Hopefully somedata we'll have a statlite()
* and can limit the fields we require to be accurate.
*/
-int ceph_getattr(struct vfsmount *mnt, struct dentry *dentry,
- struct kstat *stat)
+int ceph_getattr(const struct path *path, struct kstat *stat,
+ u32 request_mask, unsigned int flags)
{
- struct inode *inode = d_inode(dentry);
+ struct inode *inode = d_inode(path->dentry);
struct ceph_inode_info *ci = ceph_inode(inode);
int err;
extern int ceph_permission(struct inode *inode, int mask);
extern int __ceph_setattr(struct inode *inode, struct iattr *attr);
extern int ceph_setattr(struct dentry *dentry, struct iattr *attr);
-extern int ceph_getattr(struct vfsmount *mnt, struct dentry *dentry,
- struct kstat *stat);
+extern int ceph_getattr(const struct path *path, struct kstat *stat,
+ u32 request_mask, unsigned int flags);
/* xattr.c */
int __ceph_setxattr(struct inode *, const char *, const void *, size_t, int);
* gives us the latter, so we must adjust the result.
*/
mnt = ERR_PTR(-ENOMEM);
- full_path = build_path_from_dentry(mntpt);
+
+ /* always use tree name prefix */
+ full_path = build_path_from_dentry_optional_prefix(mntpt, true);
if (full_path == NULL)
goto cdda_exit;
* Returns:
* Address of the first string
*/
-static inline wchar_t *
-UniStrcat(wchar_t *ucs1, const wchar_t *ucs2)
+static inline __le16 *
+UniStrcat(__le16 *ucs1, const __le16 *ucs2)
{
- wchar_t *anchor = ucs1; /* save a pointer to start of ucs1 */
+ __le16 *anchor = ucs1; /* save a pointer to start of ucs1 */
while (*ucs1++) ; /* To end of first string */
ucs1--; /* Return to the null */
extern int cifs_invalidate_mapping(struct inode *inode);
extern int cifs_revalidate_mapping(struct inode *inode);
extern int cifs_zap_mapping(struct inode *inode);
-extern int cifs_getattr(struct vfsmount *, struct dentry *, struct kstat *);
+extern int cifs_getattr(const struct path *, struct kstat *, u32, unsigned int);
extern int cifs_setattr(struct dentry *, struct iattr *);
extern const struct inode_operations cifs_file_inode_ops;
int (*is_transform_hdr)(void *buf);
int (*receive_transform)(struct TCP_Server_Info *,
struct mid_q_entry **);
+ enum securityEnum (*select_sectype)(struct TCP_Server_Info *,
+ enum securityEnum);
+
};
struct smb_version_values {
int ses_count; /* reference counter */
enum statusEnum status;
unsigned overrideSecFlg; /* if non-zero override global sec flags */
- __u16 ipc_tid; /* special tid for connection to IPC share */
+ __u32 ipc_tid; /* special tid for connection to IPC share */
char *serverOS; /* name of operating system underlying server */
char *serverNOS; /* name of network operating system of server */
char *serverDomain; /* security realm of server */
__u8 ServiceSiteGuid[16]; /* MBZ, ignored */
} __attribute__((packed)) REFERRAL3;
-typedef struct smb_com_transaction_get_dfs_refer_rsp {
- struct smb_hdr hdr; /* wct = 10 */
- struct trans2_resp t2;
- __u16 ByteCount;
- __u8 Pad;
+struct get_dfs_referral_rsp {
__le16 PathConsumed;
__le16 NumberOfReferrals;
__le32 DFSFlags;
REFERRAL3 referrals[1]; /* array of level 3 dfs_referral structures */
/* followed by the strings pointed to by the referral structures */
-} __attribute__((packed)) TRANSACTION2_GET_DFS_REFER_RSP;
+} __packed;
+
+typedef struct smb_com_transaction_get_dfs_refer_rsp {
+ struct smb_hdr hdr; /* wct = 10 */
+ struct trans2_resp t2;
+ __u16 ByteCount;
+ __u8 Pad;
+ struct get_dfs_referral_rsp dfs_data;
+} __packed TRANSACTION2_GET_DFS_REFER_RSP;
/* DFS Flags */
#define DFSREF_REFERRAL_SERVER 0x00000001 /* all targets are DFS roots */
extern int init_cifs_spnego(void);
extern void exit_cifs_spnego(void);
extern char *build_path_from_dentry(struct dentry *);
+extern char *build_path_from_dentry_optional_prefix(struct dentry *direntry,
+ bool prefix);
extern char *cifs_build_path_to_root(struct smb_vol *vol,
struct cifs_sb_info *cifs_sb,
struct cifs_tcon *tcon,
const struct nls_table *nls_codepage,
unsigned int *num_referrals,
struct dfs_info3_param **referrals, int remap);
+extern int parse_dfs_referrals(struct get_dfs_referral_rsp *rsp, u32 rsp_size,
+ unsigned int *num_of_nodes,
+ struct dfs_info3_param **target_nodes,
+ const struct nls_table *nls_codepage, int remap,
+ const char *searchName, bool is_unicode);
extern void reset_cifs_unix_caps(unsigned int xid, struct cifs_tcon *tcon,
struct cifs_sb_info *cifs_sb,
struct smb_vol *vol);
int __cifs_calc_signature(struct smb_rqst *rqst,
struct TCP_Server_Info *server, char *signature,
struct shash_desc *shash);
+enum securityEnum cifs_select_sectype(struct TCP_Server_Info *,
+ enum securityEnum);
#endif /* _CIFSPROTO_H */
return rc;
}
-/* parses DFS refferal V3 structure
- * caller is responsible for freeing target_nodes
- * returns:
- * on success - 0
- * on failure - errno
- */
-static int
-parse_DFS_referrals(TRANSACTION2_GET_DFS_REFER_RSP *pSMBr,
- unsigned int *num_of_nodes,
- struct dfs_info3_param **target_nodes,
- const struct nls_table *nls_codepage, int remap,
- const char *searchName)
-{
- int i, rc = 0;
- char *data_end;
- bool is_unicode;
- struct dfs_referral_level_3 *ref;
-
- if (pSMBr->hdr.Flags2 & SMBFLG2_UNICODE)
- is_unicode = true;
- else
- is_unicode = false;
- *num_of_nodes = le16_to_cpu(pSMBr->NumberOfReferrals);
-
- if (*num_of_nodes < 1) {
- cifs_dbg(VFS, "num_referrals: must be at least > 0, but we get num_referrals = %d\n",
- *num_of_nodes);
- rc = -EINVAL;
- goto parse_DFS_referrals_exit;
- }
-
- ref = (struct dfs_referral_level_3 *) &(pSMBr->referrals);
- if (ref->VersionNumber != cpu_to_le16(3)) {
- cifs_dbg(VFS, "Referrals of V%d version are not supported, should be V3\n",
- le16_to_cpu(ref->VersionNumber));
- rc = -EINVAL;
- goto parse_DFS_referrals_exit;
- }
-
- /* get the upper boundary of the resp buffer */
- data_end = (char *)(&(pSMBr->PathConsumed)) +
- le16_to_cpu(pSMBr->t2.DataCount);
-
- cifs_dbg(FYI, "num_referrals: %d dfs flags: 0x%x ...\n",
- *num_of_nodes, le32_to_cpu(pSMBr->DFSFlags));
-
- *target_nodes = kcalloc(*num_of_nodes, sizeof(struct dfs_info3_param),
- GFP_KERNEL);
- if (*target_nodes == NULL) {
- rc = -ENOMEM;
- goto parse_DFS_referrals_exit;
- }
-
- /* collect necessary data from referrals */
- for (i = 0; i < *num_of_nodes; i++) {
- char *temp;
- int max_len;
- struct dfs_info3_param *node = (*target_nodes)+i;
-
- node->flags = le32_to_cpu(pSMBr->DFSFlags);
- if (is_unicode) {
- __le16 *tmp = kmalloc(strlen(searchName)*2 + 2,
- GFP_KERNEL);
- if (tmp == NULL) {
- rc = -ENOMEM;
- goto parse_DFS_referrals_exit;
- }
- cifsConvertToUTF16((__le16 *) tmp, searchName,
- PATH_MAX, nls_codepage, remap);
- node->path_consumed = cifs_utf16_bytes(tmp,
- le16_to_cpu(pSMBr->PathConsumed),
- nls_codepage);
- kfree(tmp);
- } else
- node->path_consumed = le16_to_cpu(pSMBr->PathConsumed);
-
- node->server_type = le16_to_cpu(ref->ServerType);
- node->ref_flag = le16_to_cpu(ref->ReferralEntryFlags);
-
- /* copy DfsPath */
- temp = (char *)ref + le16_to_cpu(ref->DfsPathOffset);
- max_len = data_end - temp;
- node->path_name = cifs_strndup_from_utf16(temp, max_len,
- is_unicode, nls_codepage);
- if (!node->path_name) {
- rc = -ENOMEM;
- goto parse_DFS_referrals_exit;
- }
-
- /* copy link target UNC */
- temp = (char *)ref + le16_to_cpu(ref->NetworkAddressOffset);
- max_len = data_end - temp;
- node->node_name = cifs_strndup_from_utf16(temp, max_len,
- is_unicode, nls_codepage);
- if (!node->node_name) {
- rc = -ENOMEM;
- goto parse_DFS_referrals_exit;
- }
-
- ref++;
- }
-
-parse_DFS_referrals_exit:
- if (rc) {
- free_dfs_info_array(*target_nodes, *num_of_nodes);
- *target_nodes = NULL;
- *num_of_nodes = 0;
- }
- return rc;
-}
-
int
CIFSGetDFSRefer(const unsigned int xid, struct cifs_ses *ses,
const char *search_name, struct dfs_info3_param **target_nodes,
get_bcc(&pSMBr->hdr), le16_to_cpu(pSMBr->t2.DataOffset));
/* parse returned result into more usable form */
- rc = parse_DFS_referrals(pSMBr, num_of_nodes,
- target_nodes, nls_codepage, remap,
- search_name);
+ rc = parse_dfs_referrals(&pSMBr->dfs_data,
+ le16_to_cpu(pSMBr->t2.DataCount),
+ num_of_nodes, target_nodes, nls_codepage,
+ remap, search_name,
+ (pSMBr->hdr.Flags2 & SMBFLG2_UNICODE) != 0);
GetDFSRefExit:
cifs_buf_release(pSMB);
#include <linux/fs.h>
#include <linux/net.h>
#include <linux/string.h>
+#include <linux/sched/signal.h>
#include <linux/list.h>
#include <linux/wait.h>
#include <linux/slab.h>
* that was specified, or "Unspecified" if that sectype was not
* compatible with the given NEGOTIATE request.
*/
- if (select_sectype(server, vol->sectype) == Unspecified)
+ if (server->ops->select_sectype(server, vol->sectype)
+ == Unspecified)
return false;
/*
}
down_read(&key->sem);
- upayload = user_key_payload(key);
+ upayload = user_key_payload_locked(key);
if (IS_ERR_OR_NULL(upayload)) {
rc = upayload ? PTR_ERR(upayload) : -EINVAL;
goto out_key_put;
/* Note: caller must free return buffer */
char *
build_path_from_dentry(struct dentry *direntry)
+{
+ struct cifs_sb_info *cifs_sb = CIFS_SB(direntry->d_sb);
+ struct cifs_tcon *tcon = cifs_sb_master_tcon(cifs_sb);
+ bool prefix = tcon->Flags & SMB_SHARE_IS_IN_DFS;
+
+ return build_path_from_dentry_optional_prefix(direntry,
+ prefix);
+}
+
+char *
+build_path_from_dentry_optional_prefix(struct dentry *direntry, bool prefix)
{
struct dentry *temp;
int namelen;
unsigned seq;
dirsep = CIFS_DIR_SEP(cifs_sb);
- if (tcon->Flags & SMB_SHARE_IS_IN_DFS)
+ if (prefix)
dfsplen = strnlen(tcon->treeName, MAX_TREE_SIZE + 1);
else
dfsplen = 0;
#include <linux/slab.h>
#include <linux/pagemap.h>
#include <linux/freezer.h>
+#include <linux/sched/signal.h>
+
#include <asm/div64.h>
#include "cifsfs.h"
#include "cifspdu.h"
return cifs_revalidate_mapping(inode);
}
-int cifs_getattr(struct vfsmount *mnt, struct dentry *dentry,
- struct kstat *stat)
+int cifs_getattr(const struct path *path, struct kstat *stat,
+ u32 request_mask, unsigned int flags)
{
+ struct dentry *dentry = path->dentry;
struct cifs_sb_info *cifs_sb = CIFS_SB(dentry->d_sb);
struct cifs_tcon *tcon = cifs_sb_master_tcon(cifs_sb);
struct inode *inode = d_inode(dentry);
cifs_add_pending_open_locked(fid, tlink, open);
spin_unlock(&tlink_tcon(open->tlink)->open_file_lock);
}
+
+/* parses DFS refferal V3 structure
+ * caller is responsible for freeing target_nodes
+ * returns:
+ * - on success - 0
+ * - on failure - errno
+ */
+int
+parse_dfs_referrals(struct get_dfs_referral_rsp *rsp, u32 rsp_size,
+ unsigned int *num_of_nodes,
+ struct dfs_info3_param **target_nodes,
+ const struct nls_table *nls_codepage, int remap,
+ const char *searchName, bool is_unicode)
+{
+ int i, rc = 0;
+ char *data_end;
+ struct dfs_referral_level_3 *ref;
+
+ *num_of_nodes = le16_to_cpu(rsp->NumberOfReferrals);
+
+ if (*num_of_nodes < 1) {
+ cifs_dbg(VFS, "num_referrals: must be at least > 0, but we get num_referrals = %d\n",
+ *num_of_nodes);
+ rc = -EINVAL;
+ goto parse_DFS_referrals_exit;
+ }
+
+ ref = (struct dfs_referral_level_3 *) &(rsp->referrals);
+ if (ref->VersionNumber != cpu_to_le16(3)) {
+ cifs_dbg(VFS, "Referrals of V%d version are not supported, should be V3\n",
+ le16_to_cpu(ref->VersionNumber));
+ rc = -EINVAL;
+ goto parse_DFS_referrals_exit;
+ }
+
+ /* get the upper boundary of the resp buffer */
+ data_end = (char *)rsp + rsp_size;
+
+ cifs_dbg(FYI, "num_referrals: %d dfs flags: 0x%x ...\n",
+ *num_of_nodes, le32_to_cpu(rsp->DFSFlags));
+
+ *target_nodes = kcalloc(*num_of_nodes, sizeof(struct dfs_info3_param),
+ GFP_KERNEL);
+ if (*target_nodes == NULL) {
+ rc = -ENOMEM;
+ goto parse_DFS_referrals_exit;
+ }
+
+ /* collect necessary data from referrals */
+ for (i = 0; i < *num_of_nodes; i++) {
+ char *temp;
+ int max_len;
+ struct dfs_info3_param *node = (*target_nodes)+i;
+
+ node->flags = le32_to_cpu(rsp->DFSFlags);
+ if (is_unicode) {
+ __le16 *tmp = kmalloc(strlen(searchName)*2 + 2,
+ GFP_KERNEL);
+ if (tmp == NULL) {
+ rc = -ENOMEM;
+ goto parse_DFS_referrals_exit;
+ }
+ cifsConvertToUTF16((__le16 *) tmp, searchName,
+ PATH_MAX, nls_codepage, remap);
+ node->path_consumed = cifs_utf16_bytes(tmp,
+ le16_to_cpu(rsp->PathConsumed),
+ nls_codepage);
+ kfree(tmp);
+ } else
+ node->path_consumed = le16_to_cpu(rsp->PathConsumed);
+
+ node->server_type = le16_to_cpu(ref->ServerType);
+ node->ref_flag = le16_to_cpu(ref->ReferralEntryFlags);
+
+ /* copy DfsPath */
+ temp = (char *)ref + le16_to_cpu(ref->DfsPathOffset);
+ max_len = data_end - temp;
+ node->path_name = cifs_strndup_from_utf16(temp, max_len,
+ is_unicode, nls_codepage);
+ if (!node->path_name) {
+ rc = -ENOMEM;
+ goto parse_DFS_referrals_exit;
+ }
+
+ /* copy link target UNC */
+ temp = (char *)ref + le16_to_cpu(ref->NetworkAddressOffset);
+ max_len = data_end - temp;
+ node->node_name = cifs_strndup_from_utf16(temp, max_len,
+ is_unicode, nls_codepage);
+ if (!node->node_name) {
+ rc = -ENOMEM;
+ goto parse_DFS_referrals_exit;
+ }
+
+ ref++;
+ }
+
+parse_DFS_referrals_exit:
+ if (rc) {
+ free_dfs_info_array(*target_nodes, *num_of_nodes);
+ *target_nodes = NULL;
+ *num_of_nodes = 0;
+ }
+ return rc;
+}
}
enum securityEnum
-select_sectype(struct TCP_Server_Info *server, enum securityEnum requested)
+cifs_select_sectype(struct TCP_Server_Info *server, enum securityEnum requested)
{
switch (server->negflavor) {
case CIFS_NEGFLAVOR_EXTENDED:
{
int type;
- type = select_sectype(ses->server, ses->sectype);
+ type = cifs_select_sectype(ses->server, ses->sectype);
cifs_dbg(FYI, "sess setup type %d\n", type);
if (type == Unspecified) {
cifs_dbg(VFS,
.is_read_op = cifs_is_read_op,
.wp_retry_size = cifs_wp_retry_size,
.dir_needs_close = cifs_dir_needs_close,
+ .select_sectype = cifs_select_sectype,
#ifdef CONFIG_CIFS_XATTR
.query_all_EAs = CIFSSMBQAllEAs,
.set_EA = CIFSSMBSetEA,
nr_ioctl_req.Timeout = 0; /* use server default (120 seconds) */
nr_ioctl_req.Reserved = 0;
rc = SMB2_ioctl(xid, oparms->tcon, fid->persistent_fid,
- fid->volatile_fid, FSCTL_LMR_REQUEST_RESILIENCY, true,
+ fid->volatile_fid, FSCTL_LMR_REQUEST_RESILIENCY,
+ true /* is_fsctl */, false /* use_ipc */,
(char *)&nr_ioctl_req, sizeof(nr_ioctl_req),
NULL, NULL /* no return info */);
if (rc == -EOPNOTSUPP) {
rc = SMB2_ioctl(xid, tcon, NO_FILE_ID, NO_FILE_ID,
FSCTL_QUERY_NETWORK_INTERFACE_INFO, true /* is_fsctl */,
+ false /* use_ipc */,
NULL /* no data input */, 0 /* no data input */,
(char **)&out_buf, &ret_data_len);
if (rc != 0)
rc = SMB2_ioctl(xid, tcon, persistent_fid, volatile_fid,
FSCTL_SRV_REQUEST_RESUME_KEY, true /* is_fsctl */,
+ false /* use_ipc */,
NULL, 0 /* no input */,
(char **)&res_key, &ret_data_len);
/* Request server copy to target from src identified by key */
rc = SMB2_ioctl(xid, tcon, trgtfile->fid.persistent_fid,
trgtfile->fid.volatile_fid, FSCTL_SRV_COPYCHUNK_WRITE,
- true /* is_fsctl */, (char *)pcchunk,
+ true /* is_fsctl */, false /* use_ipc */,
+ (char *)pcchunk,
sizeof(struct copychunk_ioctl), (char **)&retbuf,
&ret_data_len);
if (rc == 0) {
rc = SMB2_ioctl(xid, tcon, cfile->fid.persistent_fid,
cfile->fid.volatile_fid, FSCTL_SET_SPARSE,
- true /* is_fctl */, &setsparse, 1, NULL, NULL);
+ true /* is_fctl */, false /* use_ipc */,
+ &setsparse, 1, NULL, NULL);
if (rc) {
tcon->broken_sparse_sup = true;
cifs_dbg(FYI, "set sparse rc = %d\n", rc);
rc = SMB2_ioctl(xid, tcon, trgtfile->fid.persistent_fid,
trgtfile->fid.volatile_fid,
FSCTL_DUPLICATE_EXTENTS_TO_FILE,
- true /* is_fsctl */, (char *)&dup_ext_buf,
+ true /* is_fsctl */, false /* use_ipc */,
+ (char *)&dup_ext_buf,
sizeof(struct duplicate_extents_to_file),
NULL,
&ret_data_len);
return SMB2_ioctl(xid, tcon, cfile->fid.persistent_fid,
cfile->fid.volatile_fid,
FSCTL_SET_INTEGRITY_INFORMATION,
- true /* is_fsctl */, (char *)&integr_info,
+ true /* is_fsctl */, false /* use_ipc */,
+ (char *)&integr_info,
sizeof(struct fsctl_set_integrity_information_req),
NULL,
&ret_data_len);
rc = SMB2_ioctl(xid, tcon, cfile->fid.persistent_fid,
cfile->fid.volatile_fid,
FSCTL_SRV_ENUMERATE_SNAPSHOTS,
- true /* is_fsctl */, NULL, 0 /* no input data */,
+ true /* is_fsctl */, false /* use_ipc */,
+ NULL, 0 /* no input data */,
(char **)&retbuf,
&ret_data_len);
cifs_dbg(FYI, "enum snaphots ioctl returned %d and ret buflen is %d\n",
generate_random_uuid(fid->lease_key);
}
+static int
+smb2_get_dfs_refer(const unsigned int xid, struct cifs_ses *ses,
+ const char *search_name,
+ struct dfs_info3_param **target_nodes,
+ unsigned int *num_of_nodes,
+ const struct nls_table *nls_codepage, int remap)
+{
+ int rc;
+ __le16 *utf16_path = NULL;
+ int utf16_path_len = 0;
+ struct cifs_tcon *tcon;
+ struct fsctl_get_dfs_referral_req *dfs_req = NULL;
+ struct get_dfs_referral_rsp *dfs_rsp = NULL;
+ u32 dfs_req_size = 0, dfs_rsp_size = 0;
+
+ cifs_dbg(FYI, "smb2_get_dfs_refer path <%s>\n", search_name);
+
+ /*
+ * Use any tcon from the current session. Here, the first one.
+ */
+ spin_lock(&cifs_tcp_ses_lock);
+ tcon = list_first_entry_or_null(&ses->tcon_list, struct cifs_tcon,
+ tcon_list);
+ if (tcon)
+ tcon->tc_count++;
+ spin_unlock(&cifs_tcp_ses_lock);
+
+ if (!tcon) {
+ cifs_dbg(VFS, "session %p has no tcon available for a dfs referral request\n",
+ ses);
+ rc = -ENOTCONN;
+ goto out;
+ }
+
+ utf16_path = cifs_strndup_to_utf16(search_name, PATH_MAX,
+ &utf16_path_len,
+ nls_codepage, remap);
+ if (!utf16_path) {
+ rc = -ENOMEM;
+ goto out;
+ }
+
+ dfs_req_size = sizeof(*dfs_req) + utf16_path_len;
+ dfs_req = kzalloc(dfs_req_size, GFP_KERNEL);
+ if (!dfs_req) {
+ rc = -ENOMEM;
+ goto out;
+ }
+
+ /* Highest DFS referral version understood */
+ dfs_req->MaxReferralLevel = DFS_VERSION;
+
+ /* Path to resolve in an UTF-16 null-terminated string */
+ memcpy(dfs_req->RequestFileName, utf16_path, utf16_path_len);
+
+ do {
+ /* try first with IPC */
+ rc = SMB2_ioctl(xid, tcon, NO_FILE_ID, NO_FILE_ID,
+ FSCTL_DFS_GET_REFERRALS,
+ true /* is_fsctl */, true /* use_ipc */,
+ (char *)dfs_req, dfs_req_size,
+ (char **)&dfs_rsp, &dfs_rsp_size);
+ if (rc == -ENOTCONN) {
+ /* try with normal tcon */
+ rc = SMB2_ioctl(xid, tcon, NO_FILE_ID, NO_FILE_ID,
+ FSCTL_DFS_GET_REFERRALS,
+ true /* is_fsctl */, false /*use_ipc*/,
+ (char *)dfs_req, dfs_req_size,
+ (char **)&dfs_rsp, &dfs_rsp_size);
+ }
+ } while (rc == -EAGAIN);
+
+ if (rc) {
+ cifs_dbg(VFS, "ioctl error in smb2_get_dfs_refer rc=%d\n", rc);
+ goto out;
+ }
+
+ rc = parse_dfs_referrals(dfs_rsp, dfs_rsp_size,
+ num_of_nodes, target_nodes,
+ nls_codepage, remap, search_name,
+ true /* is_unicode */);
+ if (rc) {
+ cifs_dbg(VFS, "parse error in smb2_get_dfs_refer rc=%d\n", rc);
+ goto out;
+ }
+
+ out:
+ if (tcon) {
+ spin_lock(&cifs_tcp_ses_lock);
+ tcon->tc_count--;
+ spin_unlock(&cifs_tcp_ses_lock);
+ }
+ kfree(utf16_path);
+ kfree(dfs_req);
+ kfree(dfs_rsp);
+ return rc;
+}
#define SMB2_SYMLINK_STRUCT_SIZE \
(sizeof(struct smb2_err_rsp) - 1 + sizeof(struct smb2_symlink_err_rsp))
rc = SMB2_ioctl(xid, tcon, cfile->fid.persistent_fid,
cfile->fid.volatile_fid, FSCTL_SET_ZERO_DATA,
- true /* is_fctl */, (char *)&fsctl_buf,
+ true /* is_fctl */, false /* use_ipc */,
+ (char *)&fsctl_buf,
sizeof(struct file_zero_data_information), NULL, NULL);
free_xid(xid);
return rc;
rc = SMB2_ioctl(xid, tcon, cfile->fid.persistent_fid,
cfile->fid.volatile_fid, FSCTL_SET_ZERO_DATA,
- true /* is_fctl */, (char *)&fsctl_buf,
+ true /* is_fctl */, false /* use_ipc */,
+ (char *)&fsctl_buf,
sizeof(struct file_zero_data_information), NULL, NULL);
free_xid(xid);
return rc;
complete(&res->completion);
}
+static int
+smb2_get_enc_key(struct TCP_Server_Info *server, __u64 ses_id, int enc, u8 *key)
+{
+ struct cifs_ses *ses;
+ u8 *ses_enc_key;
+
+ spin_lock(&cifs_tcp_ses_lock);
+ list_for_each_entry(ses, &server->smb_ses_list, smb_ses_list) {
+ if (ses->Suid != ses_id)
+ continue;
+ ses_enc_key = enc ? ses->smb3encryptionkey :
+ ses->smb3decryptionkey;
+ memcpy(key, ses_enc_key, SMB3_SIGN_KEY_SIZE);
+ spin_unlock(&cifs_tcp_ses_lock);
+ return 0;
+ }
+ spin_unlock(&cifs_tcp_ses_lock);
+
+ return 1;
+}
/*
* Encrypt or decrypt @rqst message. @rqst has the following format:
* iov[0] - transform header (associate data),
struct smb2_transform_hdr *tr_hdr =
(struct smb2_transform_hdr *)rqst->rq_iov[0].iov_base;
unsigned int assoc_data_len = sizeof(struct smb2_transform_hdr) - 24;
- struct cifs_ses *ses;
int rc = 0;
struct scatterlist *sg;
u8 sign[SMB2_SIGNATURE_SIZE] = {};
+ u8 key[SMB3_SIGN_KEY_SIZE];
struct aead_request *req;
char *iv;
unsigned int iv_len;
init_completion(&result.completion);
- ses = smb2_find_smb_ses(server, tr_hdr->SessionId);
- if (!ses) {
- cifs_dbg(VFS, "%s: Could not find session\n", __func__);
+ rc = smb2_get_enc_key(server, tr_hdr->SessionId, enc, key);
+ if (rc) {
+ cifs_dbg(VFS, "%s: Could not get %scryption key\n", __func__,
+ enc ? "en" : "de");
return 0;
}
tfm = enc ? server->secmech.ccmaesencrypt :
server->secmech.ccmaesdecrypt;
- rc = crypto_aead_setkey(tfm, enc ? ses->smb3encryptionkey :
- ses->smb3decryptionkey, SMB3_SIGN_KEY_SIZE);
+ rc = crypto_aead_setkey(tfm, key, SMB3_SIGN_KEY_SIZE);
if (rc) {
cifs_dbg(VFS, "%s: Failed to set aead key %d\n", __func__, rc);
return rc;
.clone_range = smb2_clone_range,
.wp_retry_size = smb2_wp_retry_size,
.dir_needs_close = smb2_dir_needs_close,
+ .get_dfs_refer = smb2_get_dfs_refer,
+ .select_sectype = smb2_select_sectype,
};
struct smb_version_operations smb21_operations = {
.wp_retry_size = smb2_wp_retry_size,
.dir_needs_close = smb2_dir_needs_close,
.enum_snapshots = smb3_enum_snapshots,
+ .get_dfs_refer = smb2_get_dfs_refer,
+ .select_sectype = smb2_select_sectype,
};
struct smb_version_operations smb30_operations = {
.free_transform_rq = smb3_free_transform_rq,
.is_transform_hdr = smb3_is_transform_hdr,
.receive_transform = smb3_receive_transform,
+ .get_dfs_refer = smb2_get_dfs_refer,
+ .select_sectype = smb2_select_sectype,
};
#ifdef CONFIG_CIFS_SMB311
.free_transform_rq = smb3_free_transform_rq,
.is_transform_hdr = smb3_is_transform_hdr,
.receive_transform = smb3_receive_transform,
+ .get_dfs_refer = smb2_get_dfs_refer,
+ .select_sectype = smb2_select_sectype,
};
#endif /* CIFS_SMB311 */
rc = SMB2_ioctl(xid, tcon, NO_FILE_ID, NO_FILE_ID,
FSCTL_VALIDATE_NEGOTIATE_INFO, true /* is_fsctl */,
+ false /* use_ipc */,
(char *)&vneg_inbuf, sizeof(struct validate_negotiate_info_req),
(char **)&pneg_rsp, &rsplen);
return -EIO;
}
+enum securityEnum
+smb2_select_sectype(struct TCP_Server_Info *server, enum securityEnum requested)
+{
+ switch (requested) {
+ case Kerberos:
+ case RawNTLMSSP:
+ return requested;
+ case NTLMv2:
+ return RawNTLMSSP;
+ case Unspecified:
+ if (server->sec_ntlmssp &&
+ (global_secflags & CIFSSEC_MAY_NTLMSSP))
+ return RawNTLMSSP;
+ if ((server->sec_kerberos || server->sec_mskerberos) &&
+ (global_secflags & CIFSSEC_MAY_KRB5))
+ return Kerberos;
+ /* Fallthrough */
+ default:
+ return Unspecified;
+ }
+}
+
struct SMB2_sess_data {
unsigned int xid;
struct cifs_ses *ses;
static int
SMB2_select_sec(struct cifs_ses *ses, struct SMB2_sess_data *sess_data)
{
- if (ses->sectype != Kerberos && ses->sectype != RawNTLMSSP)
- ses->sectype = RawNTLMSSP;
+ int type;
+
+ type = smb2_select_sectype(ses->server, ses->sectype);
+ cifs_dbg(FYI, "sess setup type %d\n", type);
+ if (type == Unspecified) {
+ cifs_dbg(VFS,
+ "Unable to select appropriate authentication method!");
+ return -EINVAL;
+ }
- switch (ses->sectype) {
+ switch (type) {
case Kerberos:
sess_data->func = SMB2_auth_kerberos;
break;
sess_data->func = SMB2_sess_auth_rawntlmssp_negotiate;
break;
default:
- cifs_dbg(VFS, "secType %d not supported!\n", ses->sectype);
+ cifs_dbg(VFS, "secType %d not supported!\n", type);
return -EOPNOTSUPP;
}
/* since no tcon, smb2_init can not do this, so do here */
req->hdr.sync_hdr.SessionId = ses->Suid;
- /* if (ses->server->sec_mode & SECMODE_SIGN_REQUIRED)
- req->hdr.Flags |= SMB2_FLAGS_SIGNED; */
+ if (ses->server->sign)
+ req->hdr.sync_hdr.Flags |= SMB2_FLAGS_SIGNED;
} else if (encryption_required(tcon))
flags |= CIFS_TRANSFORM_REQ;
return 0;
}
+static int
+alloc_path_with_tree_prefix(__le16 **out_path, int *out_size, int *out_len,
+ const char *treename, const __le16 *path)
+{
+ int treename_len, path_len;
+ struct nls_table *cp;
+ const __le16 sep[] = {cpu_to_le16('\\'), cpu_to_le16(0x0000)};
+
+ /*
+ * skip leading "\\"
+ */
+ treename_len = strlen(treename);
+ if (treename_len < 2 || !(treename[0] == '\\' && treename[1] == '\\'))
+ return -EINVAL;
+
+ treename += 2;
+ treename_len -= 2;
+
+ path_len = UniStrnlen((wchar_t *)path, PATH_MAX);
+
+ /*
+ * make room for one path separator between the treename and
+ * path
+ */
+ *out_len = treename_len + 1 + path_len;
+
+ /*
+ * final path needs to be null-terminated UTF16 with a
+ * size aligned to 8
+ */
+
+ *out_size = roundup((*out_len+1)*2, 8);
+ *out_path = kzalloc(*out_size, GFP_KERNEL);
+ if (!*out_path)
+ return -ENOMEM;
+
+ cp = load_nls_default();
+ cifs_strtoUTF16(*out_path, treename, treename_len, cp);
+ UniStrcat(*out_path, sep);
+ UniStrcat(*out_path, path);
+ unload_nls(cp);
+
+ return 0;
+}
+
int
SMB2_open(const unsigned int xid, struct cifs_open_parms *oparms, __le16 *path,
__u8 *oplock, struct smb2_file_all_info *buf,
req->ShareAccess = FILE_SHARE_ALL_LE;
req->CreateDisposition = cpu_to_le32(oparms->disposition);
req->CreateOptions = cpu_to_le32(oparms->create_options & CREATE_OPTIONS_MASK);
- uni_path_len = (2 * UniStrnlen((wchar_t *)path, PATH_MAX)) + 2;
- /* do not count rfc1001 len field */
- req->NameOffset = cpu_to_le16(sizeof(struct smb2_create_req) - 4);
iov[0].iov_base = (char *)req;
/* 4 for rfc1002 length field */
iov[0].iov_len = get_rfc1002_length(req) + 4;
-
- /* MUST set path len (NameLength) to 0 opening root of share */
- req->NameLength = cpu_to_le16(uni_path_len - 2);
/* -1 since last byte is buf[0] which is sent below (path) */
iov[0].iov_len--;
- if (uni_path_len % 8 != 0) {
- copy_size = uni_path_len / 8 * 8;
- if (copy_size < uni_path_len)
- copy_size += 8;
-
- copy_path = kzalloc(copy_size, GFP_KERNEL);
- if (!copy_path)
- return -ENOMEM;
- memcpy((char *)copy_path, (const char *)path,
- uni_path_len);
+
+ req->NameOffset = cpu_to_le16(sizeof(struct smb2_create_req) - 4);
+
+ /* [MS-SMB2] 2.2.13 NameOffset:
+ * If SMB2_FLAGS_DFS_OPERATIONS is set in the Flags field of
+ * the SMB2 header, the file name includes a prefix that will
+ * be processed during DFS name normalization as specified in
+ * section 3.3.5.9. Otherwise, the file name is relative to
+ * the share that is identified by the TreeId in the SMB2
+ * header.
+ */
+ if (tcon->share_flags & SHI1005_FLAGS_DFS) {
+ int name_len;
+
+ req->hdr.sync_hdr.Flags |= SMB2_FLAGS_DFS_OPERATIONS;
+ rc = alloc_path_with_tree_prefix(©_path, ©_size,
+ &name_len,
+ tcon->treeName, path);
+ if (rc)
+ return rc;
+ req->NameLength = cpu_to_le16(name_len * 2);
uni_path_len = copy_size;
path = copy_path;
+ } else {
+ uni_path_len = (2 * UniStrnlen((wchar_t *)path, PATH_MAX)) + 2;
+ /* MUST set path len (NameLength) to 0 opening root of share */
+ req->NameLength = cpu_to_le16(uni_path_len - 2);
+ if (uni_path_len % 8 != 0) {
+ copy_size = roundup(uni_path_len, 8);
+ copy_path = kzalloc(copy_size, GFP_KERNEL);
+ if (!copy_path)
+ return -ENOMEM;
+ memcpy((char *)copy_path, (const char *)path,
+ uni_path_len);
+ uni_path_len = copy_size;
+ path = copy_path;
+ }
}
iov[1].iov_len = uni_path_len;
*/
int
SMB2_ioctl(const unsigned int xid, struct cifs_tcon *tcon, u64 persistent_fid,
- u64 volatile_fid, u32 opcode, bool is_fsctl, char *in_data,
- u32 indatalen, char **out_data, u32 *plen /* returned data len */)
+ u64 volatile_fid, u32 opcode, bool is_fsctl, bool use_ipc,
+ char *in_data, u32 indatalen,
+ char **out_data, u32 *plen /* returned data len */)
{
struct smb2_ioctl_req *req;
struct smb2_ioctl_rsp *rsp;
if (rc)
return rc;
+ if (use_ipc) {
+ if (ses->ipc_tid == 0) {
+ cifs_small_buf_release(req);
+ return -ENOTCONN;
+ }
+
+ cifs_dbg(FYI, "replacing tid 0x%x with IPC tid 0x%x\n",
+ req->hdr.sync_hdr.TreeId, ses->ipc_tid);
+ req->hdr.sync_hdr.TreeId = ses->ipc_tid;
+ }
if (encryption_required(tcon))
flags |= CIFS_TRANSFORM_REQ;
rc = SMB2_ioctl(xid, tcon, persistent_fid, volatile_fid,
FSCTL_SET_COMPRESSION, true /* is_fsctl */,
+ false /* use_ipc */,
(char *)&fsctl_input /* data input */,
2 /* in data len */, &ret_data /* out data */, NULL);
/* Integrity flags for above */
#define FSCTL_INTEGRITY_FLAG_CHECKSUM_ENFORCEMENT_OFF 0x00000001
+/* See MS-DFSC 2.2.2 */
+struct fsctl_get_dfs_referral_req {
+ __le16 MaxReferralLevel;
+ __u8 RequestFileName[];
+} __packed;
+
+/* DFS response is struct get_dfs_refer_rsp */
+
/* See MS-SMB2 2.2.31.3 */
struct network_resiliency_req {
__le32 Timeout;
struct smb2_err_rsp **err_buf);
extern int SMB2_ioctl(const unsigned int xid, struct cifs_tcon *tcon,
u64 persistent_fid, u64 volatile_fid, u32 opcode,
- bool is_fsctl, char *in_data, u32 indatalen,
+ bool is_fsctl, bool use_ipc,
+ char *in_data, u32 indatalen,
char **out_data, u32 *plen /* returned data len */);
extern int SMB2_close(const unsigned int xid, struct cifs_tcon *tcon,
u64 persistent_file_id, u64 volatile_file_id);
__u8 *lease_key, const __le32 lease_state);
extern int smb3_validate_negotiate(const unsigned int, struct cifs_tcon *);
+extern enum securityEnum smb2_select_sectype(struct TCP_Server_Info *,
+ enum securityEnum);
#endif /* _SMB2PROTO_H */
int coda_release(struct inode *i, struct file *f);
int coda_permission(struct inode *inode, int mask);
int coda_revalidate_inode(struct inode *);
-int coda_getattr(struct vfsmount *, struct dentry *, struct kstat *);
+int coda_getattr(const struct path *, struct kstat *, u32, unsigned int);
int coda_setattr(struct dentry *, struct iattr *);
/* this file: heloers */
cfi->cfi_mapcount++;
spin_unlock(&cii->c_lock);
- return host_file->f_op->mmap(host_file, vma);
+ return call_mmap(host_file, vma);
}
int coda_open(struct inode *coda_inode, struct file *coda_file)
coda_cache_clear_inode(inode);
}
-int coda_getattr(struct vfsmount *mnt, struct dentry *dentry, struct kstat *stat)
+int coda_getattr(const struct path *path, struct kstat *stat,
+ u32 request_mask, unsigned int flags)
{
- int err = coda_revalidate_inode(d_inode(dentry));
+ int err = coda_revalidate_inode(d_inode(path->dentry));
if (!err)
- generic_fillattr(d_inode(dentry), stat);
+ generic_fillattr(d_inode(path->dentry), stat);
return err;
}
#include <linux/kernel.h>
#include <linux/major.h>
#include <linux/time.h>
-#include <linux/sched.h>
+#include <linux/sched/signal.h>
#include <linux/slab.h>
#include <linux/ioport.h>
#include <linux/fcntl.h>
*/
#include <linux/signal.h>
-#include <linux/sched.h>
+#include <linux/sched/signal.h>
#include <linux/types.h>
#include <linux/kernel.h>
#include <linux/mm.h>
#include <linux/compat.h>
#include <linux/errno.h>
#include <linux/time.h>
+#include <linux/cred.h>
#include <linux/fs.h>
#include <linux/fcntl.h>
#include <linux/namei.h>
#include <linux/personality.h>
#include <linux/binfmts.h>
#include <linux/coredump.h>
+#include <linux/sched/coredump.h>
+#include <linux/sched/signal.h>
+#include <linux/sched/task_stack.h>
#include <linux/utsname.h>
#include <linux/pid_namespace.h>
#include <linux/module.h>
#include <linux/pipe_fs_i.h>
#include <linux/oom.h>
#include <linux/compat.h>
-#include <linux/sched.h>
#include <linux/fs.h>
#include <linux/path.h>
#include <linux/timekeeping.h>
goto out;
}
down_read(&keyring_key->sem);
- ukp = user_key_payload(keyring_key);
+ ukp = user_key_payload_locked(keyring_key);
if (ukp->datalen != sizeof(struct fscrypt_key)) {
res = -EINVAL;
up_read(&keyring_key->sem);
#include <linux/pagevec.h>
#include <linux/pmem.h>
#include <linux/sched.h>
+#include <linux/sched/signal.h>
#include <linux/uio.h>
#include <linux/vmstat.h>
#include <linux/pfn_t.h>
#include <linux/dlm.h>
#include <linux/dlm_device.h>
#include <linux/slab.h>
+#include <linux/sched/signal.h>
#include "dlm_internal.h"
#include "lockspace.h"
auth_tok = ecryptfs_get_encrypted_key_payload_data(key);
if (!auth_tok)
- return (struct ecryptfs_auth_tok *)user_key_payload(key)->data;
+ return (struct ecryptfs_auth_tok *)user_key_payload_locked(key)->data;
else
return auth_tok;
}
return rc;
}
-static int ecryptfs_getattr_link(struct vfsmount *mnt, struct dentry *dentry,
- struct kstat *stat)
+static int ecryptfs_getattr_link(const struct path *path, struct kstat *stat,
+ u32 request_mask, unsigned int flags)
{
+ struct dentry *dentry = path->dentry;
struct ecryptfs_mount_crypt_stat *mount_crypt_stat;
int rc = 0;
return rc;
}
-static int ecryptfs_getattr(struct vfsmount *mnt, struct dentry *dentry,
- struct kstat *stat)
+static int ecryptfs_getattr(const struct path *path, struct kstat *stat,
+ u32 request_mask, unsigned int flags)
{
+ struct dentry *dentry = path->dentry;
struct kstat lower_stat;
int rc;
- rc = vfs_getattr(ecryptfs_dentry_to_lower_path(dentry), &lower_stat);
+ rc = vfs_getattr(ecryptfs_dentry_to_lower_path(dentry), &lower_stat,
+ request_mask, flags);
if (!rc) {
fsstack_copy_attr_all(d_inode(dentry),
ecryptfs_inode_to_lower(d_inode(dentry)));
#include <linux/fs.h>
#include <linux/pagemap.h>
+#include <linux/sched/signal.h>
+
#include "ecryptfs_kernel.h"
/**
#include <linux/poll.h>
#include <linux/init.h>
#include <linux/fs.h>
-#include <linux/sched.h>
+#include <linux/sched/signal.h>
#include <linux/kernel.h>
#include <linux/slab.h>
#include <linux/list.h>
#include <linux/init.h>
#include <linux/kernel.h>
-#include <linux/sched.h>
+#include <linux/sched/signal.h>
#include <linux/fs.h>
#include <linux/file.h>
#include <linux/signal.h>
#include <linux/swap.h>
#include <linux/string.h>
#include <linux/init.h>
+#include <linux/sched/mm.h>
+#include <linux/sched/coredump.h>
+#include <linux/sched/signal.h>
+#include <linux/sched/numa_balancing.h>
+#include <linux/sched/task.h>
#include <linux/pagemap.h>
#include <linux/perf_event.h>
#include <linux/highmem.h>
struct task_struct *leader = tsk->group_leader;
for (;;) {
- threadgroup_change_begin(tsk);
+ cgroup_threadgroup_change_begin(tsk);
write_lock_irq(&tasklist_lock);
/*
* Do this under tasklist_lock to ensure that
break;
__set_current_state(TASK_KILLABLE);
write_unlock_irq(&tasklist_lock);
- threadgroup_change_end(tsk);
+ cgroup_threadgroup_change_end(tsk);
schedule();
if (unlikely(__fatal_signal_pending(tsk)))
goto killed;
if (unlikely(leader->ptrace))
__wake_up_parent(leader, leader->parent);
write_unlock_irq(&tasklist_lock);
- threadgroup_change_end(tsk);
+ cgroup_threadgroup_change_end(tsk);
release_task(leader);
}
#include <linux/mount.h>
#include <linux/namei.h>
#include <linux/sched.h>
+#include <linux/cred.h>
#define dprintk(fmt, args...) do{}while(0)
* filesystem supports 64-bit inode numbers. So we need to
* actually call ->getattr, not just read i_ino:
*/
- error = vfs_getattr_nosec(&child_path, &stat);
+ error = vfs_getattr_nosec(&child_path, &stat,
+ STATX_INO, AT_STATX_SYNC_AS_STAT);
if (error)
return error;
buffer.ino = stat.ino;
#include <linux/quotaops.h>
#include <linux/slab.h>
#include <linux/sched.h>
+#include <linux/cred.h>
#include <linux/buffer_head.h>
#include <linux/capability.h>
#include <linux/timer.h>
#include <linux/version.h>
#include <linux/wait.h>
+#include <linux/sched/signal.h>
#include <linux/blockgroup_lock.h>
#include <linux/percpu_counter.h>
#include <linux/ratelimit.h>
extern struct inode *ext4_iget_normal(struct super_block *, unsigned long);
extern int ext4_write_inode(struct inode *, struct writeback_control *);
extern int ext4_setattr(struct dentry *, struct iattr *);
-extern int ext4_getattr(struct vfsmount *mnt, struct dentry *dentry,
- struct kstat *stat);
+extern int ext4_getattr(const struct path *, struct kstat *, u32, unsigned int);
extern void ext4_evict_inode(struct inode *);
extern void ext4_clear_inode(struct inode *);
extern int ext4_sync_inode(handle_t *, struct inode *);
#include <linux/random.h>
#include <linux/bitops.h>
#include <linux/blkdev.h>
+#include <linux/cred.h>
+
#include <asm/byteorder.h>
#include "ext4.h"
return error;
}
-int ext4_getattr(struct vfsmount *mnt, struct dentry *dentry,
- struct kstat *stat)
+int ext4_getattr(const struct path *path, struct kstat *stat,
+ u32 request_mask, unsigned int query_flags)
{
struct inode *inode;
unsigned long long delalloc_blocks;
- inode = d_inode(dentry);
+ inode = d_inode(path->dentry);
generic_fillattr(inode, stat);
/*
#include <linux/mm.h>
#include <linux/memcontrol.h>
#include <linux/cleancache.h>
+#include <linux/sched/signal.h>
#include "f2fs.h"
#include "node.h"
void truncate_data_blocks(struct dnode_of_data *dn);
int truncate_blocks(struct inode *inode, u64 from, bool lock);
int f2fs_truncate(struct inode *inode);
-int f2fs_getattr(struct vfsmount *mnt, struct dentry *dentry,
- struct kstat *stat);
+int f2fs_getattr(const struct path *path, struct kstat *stat,
+ u32 request_mask, unsigned int flags);
int f2fs_setattr(struct dentry *dentry, struct iattr *attr);
int truncate_hole(struct inode *inode, pgoff_t pg_start, pgoff_t pg_end);
int truncate_data_blocks_range(struct dnode_of_data *dn, int count);
return 0;
}
-int f2fs_getattr(struct vfsmount *mnt,
- struct dentry *dentry, struct kstat *stat)
+int f2fs_getattr(const struct path *path, struct kstat *stat,
+ u32 request_mask, unsigned int flags)
{
- struct inode *inode = d_inode(dentry);
+ struct inode *inode = d_inode(path->dentry);
generic_fillattr(inode, stat);
stat->blocks <<= 3;
return 0;
extern const struct inode_operations fat_file_inode_operations;
extern int fat_setattr(struct dentry *dentry, struct iattr *attr);
extern void fat_truncate_blocks(struct inode *inode, loff_t offset);
-extern int fat_getattr(struct vfsmount *mnt, struct dentry *dentry,
- struct kstat *stat);
+extern int fat_getattr(const struct path *path, struct kstat *stat,
+ u32 request_mask, unsigned int flags);
extern int fat_file_fsync(struct file *file, loff_t start, loff_t end,
int datasync);
fat_flush_inodes(inode->i_sb, inode, NULL);
}
-int fat_getattr(struct vfsmount *mnt, struct dentry *dentry, struct kstat *stat)
+int fat_getattr(const struct path *path, struct kstat *stat,
+ u32 request_mask, unsigned int flags)
{
- struct inode *inode = d_inode(dentry);
+ struct inode *inode = d_inode(path->dentry);
generic_fillattr(inode, stat);
stat->blksize = MSDOS_SB(inode->i_sb)->cluster_size;
return 0;
}
+static void fat_dummy_inode_init(struct inode *inode)
+{
+ /* Initialize this dummy inode to work as no-op. */
+ MSDOS_I(inode)->mmu_private = 0;
+ MSDOS_I(inode)->i_start = 0;
+ MSDOS_I(inode)->i_logstart = 0;
+ MSDOS_I(inode)->i_attrs = 0;
+ MSDOS_I(inode)->i_pos = 0;
+}
+
static int fat_read_root(struct inode *inode)
{
struct msdos_sb_info *sbi = MSDOS_SB(inode->i_sb);
fat_inode = new_inode(sb);
if (!fat_inode)
goto out_fail;
- MSDOS_I(fat_inode)->i_pos = 0;
+ fat_dummy_inode_init(fat_inode);
sbi->fat_inode = fat_inode;
fsinfo_inode = new_inode(sb);
if (!fsinfo_inode)
goto out_fail;
+ fat_dummy_inode_init(fsinfo_inode);
fsinfo_inode->i_ino = MSDOS_FSINFO_INO;
sbi->fsinfo_inode = fsinfo_inode;
insert_inode_hash(fsinfo_inode);
#include <linux/syscalls.h>
#include <linux/init.h>
#include <linux/mm.h>
+#include <linux/sched/task.h>
#include <linux/fs.h>
#include <linux/file.h>
#include <linux/fdtable.h>
#include <linux/mm.h>
#include <linux/mmzone.h>
#include <linux/time.h>
-#include <linux/sched.h>
+#include <linux/sched/signal.h>
#include <linux/slab.h>
#include <linux/vmalloc.h>
#include <linux/file.h>
#include <linux/module.h>
#include <linux/fs.h>
#include <linux/security.h>
+#include <linux/cred.h>
#include <linux/eventpoll.h>
#include <linux/rcupdate.h>
#include <linux/mount.h>
#include <linux/export.h>
-#include <linux/sched.h>
+#include <linux/sched/signal.h>
+#include <linux/sched/task.h>
#include <linux/fs.h>
#include <linux/path.h>
#include <linux/slab.h>
config = 0;
rcu_read_lock();
- confkey = user_key_payload(key);
+ confkey = user_key_payload_rcu(key);
buf = confkey->data;
for (len = confkey->datalen - 1; len >= 0; len--) {
#include <linux/init.h>
#include <linux/module.h>
#include <linux/poll.h>
+#include <linux/sched/signal.h>
#include <linux/uio.h>
#include <linux/miscdevice.h>
#include <linux/pagemap.h>
if (err) {
fuse_sync_release(ff, flags);
} else {
- file->private_data = fuse_file_get(ff);
+ file->private_data = ff;
fuse_finish_open(inode, file);
}
return err;
return ret;
}
-static int fuse_getattr(struct vfsmount *mnt, struct dentry *entry,
- struct kstat *stat)
+static int fuse_getattr(const struct path *path, struct kstat *stat,
+ u32 request_mask, unsigned int flags)
{
- struct inode *inode = d_inode(entry);
+ struct inode *inode = d_inode(path->dentry);
struct fuse_conn *fc = get_fuse_conn(inode);
if (!fuse_allow_current_process(fc))
}
INIT_LIST_HEAD(&ff->write_entry);
- atomic_set(&ff->count, 0);
+ atomic_set(&ff->count, 1);
RB_CLEAR_NODE(&ff->polled_node);
init_waitqueue_head(&ff->poll_wait);
kfree(ff);
}
-struct fuse_file *fuse_file_get(struct fuse_file *ff)
+static struct fuse_file *fuse_file_get(struct fuse_file *ff)
{
atomic_inc(&ff->count);
return ff;
iput(req->misc.release.inode);
fuse_put_request(ff->fc, req);
} else if (sync) {
+ __set_bit(FR_FORCE, &req->flags);
__clear_bit(FR_BACKGROUND, &req->flags);
fuse_request_send(ff->fc, req);
iput(req->misc.release.inode);
ff->open_flags &= ~FOPEN_DIRECT_IO;
ff->nodeid = nodeid;
- file->private_data = fuse_file_get(ff);
+ file->private_data = ff;
return 0;
}
void fuse_release_common(struct file *file, int opcode)
{
- struct fuse_file *ff;
- struct fuse_req *req;
-
- ff = file->private_data;
- if (unlikely(!ff))
- return;
+ struct fuse_file *ff = file->private_data;
+ struct fuse_req *req = ff->reserved_req;
- req = ff->reserved_req;
fuse_prepare_release(ff, file->f_flags, opcode);
if (ff->flock) {
void fuse_sync_release(struct fuse_file *ff, int flags)
{
- WARN_ON(atomic_read(&ff->count) > 1);
+ WARN_ON(atomic_read(&ff->count) != 1);
fuse_prepare_release(ff, flags, FUSE_RELEASE);
- __set_bit(FR_FORCE, &ff->reserved_req->flags);
- __clear_bit(FR_BACKGROUND, &ff->reserved_req->flags);
- fuse_request_send(ff->fc, ff->reserved_req);
- fuse_put_request(ff->fc, ff->reserved_req);
- kfree(ff);
+ /*
+ * iput(NULL) is a no-op and since the refcount is 1 and everything's
+ * synchronous, we are fine with not doing igrab() here"
+ */
+ fuse_file_put(ff, true);
}
EXPORT_SYMBOL_GPL(fuse_sync_release);
int fuse_open_common(struct inode *inode, struct file *file, bool isdir);
struct fuse_file *fuse_file_alloc(struct fuse_conn *fc);
-struct fuse_file *fuse_file_get(struct fuse_file *ff);
void fuse_file_free(struct fuse_file *ff);
void fuse_finish_open(struct inode *inode, struct file *file);
#include <linux/buffer_head.h>
#include <linux/namei.h>
#include <linux/mm.h>
+#include <linux/cred.h>
#include <linux/xattr.h>
#include <linux/posix_acl.h>
#include <linux/gfs2_ondisk.h>
/**
* gfs2_getattr - Read out an inode's attributes
- * @mnt: The vfsmount the inode is being accessed from
- * @dentry: The dentry to stat
+ * @path: Object to query
* @stat: The inode's stats
+ * @request_mask: Mask of STATX_xxx flags indicating the caller's interests
+ * @flags: AT_STATX_xxx setting
*
* This may be called from the VFS directly, or from within GFS2 with the
* inode locked, so we look to see if the glock is already locked and only
* Returns: errno
*/
-static int gfs2_getattr(struct vfsmount *mnt, struct dentry *dentry,
- struct kstat *stat)
+static int gfs2_getattr(const struct path *path, struct kstat *stat,
+ u32 request_mask, unsigned int flags)
{
- struct inode *inode = d_inode(dentry);
+ struct inode *inode = d_inode(path->dentry);
struct gfs2_inode *ip = GFS2_I(inode);
struct gfs2_holder gh;
int error;
#include <linux/types.h>
#include <linux/delay.h>
#include <linux/gfs2_ondisk.h>
+#include <linux/sched/signal.h>
#include "incore.h"
#include "glock.h"
#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
#include <linux/bio.h>
-#include <linux/sched.h>
+#include <linux/sched/signal.h>
#include <linux/slab.h>
#include <linux/spinlock.h>
#include <linux/completion.h>
#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
#include <linux/sched.h>
+#include <linux/cred.h>
#include <linux/spinlock.h>
#include <linux/completion.h>
#include <linux/buffer_head.h>
* Can be done after the list insertion; exclusion with
* hfs_delete_cat() is provided by directory lock.
*/
- memcpy(&rd->key, &fd.key, sizeof(struct hfs_cat_key));
+ memcpy(&rd->key, &fd.key->cat, sizeof(struct hfs_cat_key));
out:
hfs_find_exit(&fd);
return err;
#include <linux/pagemap.h>
#include <linux/mpage.h>
#include <linux/sched.h>
+#include <linux/cred.h>
#include <linux/uio.h>
#include <linux/xattr.h>
#include <linux/pagemap.h>
#include <linux/mpage.h>
#include <linux/sched.h>
+#include <linux/cred.h>
#include <linux/uio.h>
#include "hfsplus_fs.h"
#include <linux/pagemap.h>
#include <linux/buffer_head.h>
#include <linux/slab.h>
-#include <linux/sched.h>
+#include <linux/sched/signal.h>
#include <linux/blkdev.h>
#include <asm/unaligned.h>
#include <linux/thread_info.h>
#include <asm/current.h>
-#include <linux/sched.h> /* remove ASAP */
+#include <linux/sched/signal.h> /* remove ASAP */
#include <linux/falloc.h>
#include <linux/fs.h>
#include <linux/mount.h>
#include <linux/writeback.h>
#include <linux/buffer_head.h>
#include <linux/falloc.h>
+#include <linux/sched/signal.h>
+
#include "internal.h"
#include <asm/ioctls.h>
#include <linux/buffer_head.h>
#include <linux/task_io_accounting_ops.h>
#include <linux/dax.h>
+#include <linux/sched/signal.h>
+
#include "internal.h"
/*
struct address_space *mapping = iocb->ki_filp->f_mapping;
struct inode *inode = file_inode(iocb->ki_filp);
size_t count = iov_iter_count(iter);
- loff_t pos = iocb->ki_pos, end = iocb->ki_pos + count - 1, ret = 0;
+ loff_t pos = iocb->ki_pos, start = pos;
+ loff_t end = iocb->ki_pos + count - 1, ret = 0;
unsigned int flags = IOMAP_DIRECT;
struct blk_plug plug;
struct iomap_dio *dio;
}
if (mapping->nrpages) {
- ret = filemap_write_and_wait_range(mapping, iocb->ki_pos, end);
+ ret = filemap_write_and_wait_range(mapping, start, end);
if (ret)
goto out_free_dio;
ret = invalidate_inode_pages2_range(mapping,
- iocb->ki_pos >> PAGE_SHIFT, end >> PAGE_SHIFT);
+ start >> PAGE_SHIFT, end >> PAGE_SHIFT);
WARN_ON_ONCE(ret);
ret = 0;
}
__set_current_state(TASK_RUNNING);
}
+ ret = iomap_dio_complete(dio);
+
/*
* Try again to invalidate clean pages which might have been cached by
* non-direct readahead, or faulted in by get_user_pages() if the source
* this invalidation fails, tough, the write still worked...
*/
if (iov_iter_rw(iter) == WRITE && mapping->nrpages) {
- ret = invalidate_inode_pages2_range(mapping,
- iocb->ki_pos >> PAGE_SHIFT, end >> PAGE_SHIFT);
- WARN_ON_ONCE(ret);
+ int err = invalidate_inode_pages2_range(mapping,
+ start >> PAGE_SHIFT, end >> PAGE_SHIFT);
+ WARN_ON_ONCE(err);
}
- return iomap_dio_complete(dio);
+ return ret;
out_free_dio:
kfree(dio);
#include <linux/module.h>
#include <linux/slab.h>
+#include <linux/cred.h>
#include <linux/nls.h>
#include <linux/ctype.h>
#include <linux/statfs.h>
#include <linux/jffs2.h>
#include <linux/mtd/mtd.h>
#include <linux/completion.h>
-#include <linux/sched.h>
+#include <linux/sched/signal.h>
#include <linux/freezer.h>
#include <linux/kthread.h>
#include "nodelist.h"
#include <linux/capability.h>
#include <linux/kernel.h>
#include <linux/sched.h>
+#include <linux/cred.h>
#include <linux/fs.h>
#include <linux/list.h>
#include <linux/mtd/mtd.h>
#include <linux/kernel.h>
#include <linux/mtd/mtd.h>
#include <linux/compiler.h>
-#include <linux/sched.h> /* For cond_resched() */
+#include <linux/sched/signal.h>
#include "nodelist.h"
#include "debug.h"
#include <linux/slab.h>
#include <linux/poll.h>
#include <linux/pagemap.h>
-#include <linux/sched.h>
+#include <linux/sched/mm.h>
#include <linux/fsnotify.h>
#include "kernfs-internal.h"
set_nlink(inode, kn->dir.subdirs + 2);
}
-int kernfs_iop_getattr(struct vfsmount *mnt, struct dentry *dentry,
- struct kstat *stat)
+int kernfs_iop_getattr(const struct path *path, struct kstat *stat,
+ u32 request_mask, unsigned int query_flags)
{
- struct kernfs_node *kn = dentry->d_fsdata;
- struct inode *inode = d_inode(dentry);
+ struct kernfs_node *kn = path->dentry->d_fsdata;
+ struct inode *inode = d_inode(path->dentry);
mutex_lock(&kernfs_mutex);
kernfs_refresh_inode(kn, inode);
void kernfs_evict_inode(struct inode *inode);
int kernfs_iop_permission(struct inode *inode, int mask);
int kernfs_iop_setattr(struct dentry *dentry, struct iattr *iattr);
-int kernfs_iop_getattr(struct vfsmount *mnt, struct dentry *dentry,
- struct kstat *stat);
+int kernfs_iop_getattr(const struct path *path, struct kstat *stat,
+ u32 request_mask, unsigned int query_flags);
ssize_t kernfs_iop_listxattr(struct dentry *dentry, char *buf, size_t size);
/*
#include <linux/export.h>
#include <linux/pagemap.h>
#include <linux/slab.h>
+#include <linux/cred.h>
#include <linux/mount.h>
#include <linux/vfs.h>
#include <linux/quotaops.h>
#include "internal.h"
-int simple_getattr(struct vfsmount *mnt, struct dentry *dentry,
- struct kstat *stat)
+int simple_getattr(const struct path *path, struct kstat *stat,
+ u32 request_mask, unsigned int query_flags)
{
- struct inode *inode = d_inode(dentry);
+ struct inode *inode = d_inode(path->dentry);
generic_fillattr(inode, stat);
stat->blocks = inode->i_mapping->nrpages << (PAGE_SHIFT - 9);
return 0;
return ERR_PTR(-ENOENT);
}
-static int empty_dir_getattr(struct vfsmount *mnt, struct dentry *dentry,
- struct kstat *stat)
+static int empty_dir_getattr(const struct path *path, struct kstat *stat,
+ u32 request_mask, unsigned int query_flags)
{
- struct inode *inode = d_inode(dentry);
+ struct inode *inode = d_inode(path->dentry);
generic_fillattr(inode, stat);
return 0;
}
#include <linux/sysctl.h>
#include <linux/moduleparam.h>
-#include <linux/sched.h>
+#include <linux/sched/signal.h>
#include <linux/errno.h>
#include <linux/in.h>
#include <linux/uio.h>
return err;
}
-int minix_getattr(struct vfsmount *mnt, struct dentry *dentry, struct kstat *stat)
+int minix_getattr(const struct path *path, struct kstat *stat,
+ u32 request_mask, unsigned int flags)
{
- struct super_block *sb = dentry->d_sb;
- generic_fillattr(d_inode(dentry), stat);
- if (INODE_VERSION(d_inode(dentry)) == MINIX_V1)
+ struct super_block *sb = path->dentry->d_sb;
+ struct inode *inode = d_inode(path->dentry);
+
+ generic_fillattr(inode, stat);
+ if (INODE_VERSION(inode) == MINIX_V1)
stat->blocks = (BLOCK_SIZE / 512) * V1_minix_blocks(stat->size, sb);
else
stat->blocks = (sb->s_blocksize / 512) * V2_minix_blocks(stat->size, sb);
extern int minix_new_block(struct inode * inode);
extern void minix_free_block(struct inode *inode, unsigned long block);
extern unsigned long minix_count_free_blocks(struct super_block *sb);
-extern int minix_getattr(struct vfsmount *, struct dentry *, struct kstat *);
+extern int minix_getattr(const struct path *, struct kstat *, u32, unsigned int);
extern int minix_prepare_chunk(struct page *page, loff_t pos, unsigned len);
extern void V1_minix_truncate(struct inode *);
/**
* unlazy_walk - try to switch to ref-walk mode.
* @nd: nameidata pathwalk data
- * @dentry: child of nd->path.dentry or NULL
- * @seq: seq number to check dentry against
* Returns: 0 on success, -ECHILD on failure
*
- * unlazy_walk attempts to legitimize the current nd->path, nd->root and dentry
- * for ref-walk mode. @dentry must be a path found by a do_lookup call on
- * @nd or NULL. Must be called from rcu-walk context.
+ * unlazy_walk attempts to legitimize the current nd->path and nd->root
+ * for ref-walk mode.
+ * Must be called from rcu-walk context.
* Nothing should touch nameidata between unlazy_walk() failure and
* terminate_walk().
*/
-static int unlazy_walk(struct nameidata *nd, struct dentry *dentry, unsigned seq)
+static int unlazy_walk(struct nameidata *nd)
{
struct dentry *parent = nd->path.dentry;
BUG_ON(!(nd->flags & LOOKUP_RCU));
+ nd->flags &= ~LOOKUP_RCU;
+ if (unlikely(!legitimize_links(nd)))
+ goto out2;
+ if (unlikely(!legitimize_path(nd, &nd->path, nd->seq)))
+ goto out1;
+ if (nd->root.mnt && !(nd->flags & LOOKUP_ROOT)) {
+ if (unlikely(!legitimize_path(nd, &nd->root, nd->root_seq)))
+ goto out;
+ }
+ rcu_read_unlock();
+ BUG_ON(nd->inode != parent->d_inode);
+ return 0;
+
+out2:
+ nd->path.mnt = NULL;
+ nd->path.dentry = NULL;
+out1:
+ if (!(nd->flags & LOOKUP_ROOT))
+ nd->root.mnt = NULL;
+out:
+ rcu_read_unlock();
+ return -ECHILD;
+}
+
+/**
+ * unlazy_child - try to switch to ref-walk mode.
+ * @nd: nameidata pathwalk data
+ * @dentry: child of nd->path.dentry
+ * @seq: seq number to check dentry against
+ * Returns: 0 on success, -ECHILD on failure
+ *
+ * unlazy_child attempts to legitimize the current nd->path, nd->root and dentry
+ * for ref-walk mode. @dentry must be a path found by a do_lookup call on
+ * @nd. Must be called from rcu-walk context.
+ * Nothing should touch nameidata between unlazy_child() failure and
+ * terminate_walk().
+ */
+static int unlazy_child(struct nameidata *nd, struct dentry *dentry, unsigned seq)
+{
+ BUG_ON(!(nd->flags & LOOKUP_RCU));
+
nd->flags &= ~LOOKUP_RCU;
if (unlikely(!legitimize_links(nd)))
goto out2;
if (unlikely(!legitimize_mnt(nd->path.mnt, nd->m_seq)))
goto out2;
- if (unlikely(!lockref_get_not_dead(&parent->d_lockref)))
+ if (unlikely(!lockref_get_not_dead(&nd->path.dentry->d_lockref)))
goto out1;
/*
- * For a negative lookup, the lookup sequence point is the parents
- * sequence point, and it only needs to revalidate the parent dentry.
- *
- * For a positive lookup, we need to move both the parent and the
- * dentry from the RCU domain to be properly refcounted. And the
- * sequence number in the dentry validates *both* dentry counters,
- * since we checked the sequence number of the parent after we got
- * the child sequence number. So we know the parent must still
- * be valid if the child sequence number is still valid.
+ * We need to move both the parent and the dentry from the RCU domain
+ * to be properly refcounted. And the sequence number in the dentry
+ * validates *both* dentry counters, since we checked the sequence
+ * number of the parent after we got the child sequence number. So we
+ * know the parent must still be valid if the child sequence number is
*/
- if (!dentry) {
- if (read_seqcount_retry(&parent->d_seq, nd->seq))
- goto out;
- BUG_ON(nd->inode != parent->d_inode);
- } else {
- if (!lockref_get_not_dead(&dentry->d_lockref))
- goto out;
- if (read_seqcount_retry(&dentry->d_seq, seq))
- goto drop_dentry;
+ if (unlikely(!lockref_get_not_dead(&dentry->d_lockref)))
+ goto out;
+ if (unlikely(read_seqcount_retry(&dentry->d_seq, seq))) {
+ rcu_read_unlock();
+ dput(dentry);
+ goto drop_root_mnt;
}
-
/*
* Sequence counts matched. Now make sure that the root is
* still valid and get it if required.
rcu_read_unlock();
return 0;
-drop_dentry:
- rcu_read_unlock();
- dput(dentry);
- goto drop_root_mnt;
out2:
nd->path.mnt = NULL;
out1:
return -ECHILD;
}
-static int unlazy_link(struct nameidata *nd, struct path *link, unsigned seq)
-{
- if (unlikely(!legitimize_path(nd, link, seq))) {
- drop_links(nd);
- nd->depth = 0;
- nd->flags &= ~LOOKUP_RCU;
- nd->path.mnt = NULL;
- nd->path.dentry = NULL;
- if (!(nd->flags & LOOKUP_ROOT))
- nd->root.mnt = NULL;
- rcu_read_unlock();
- } else if (likely(unlazy_walk(nd, NULL, 0)) == 0) {
- return 0;
- }
- path_put(link);
- return -ECHILD;
-}
-
static inline int d_revalidate(struct dentry *dentry, unsigned int flags)
{
- return dentry->d_op->d_revalidate(dentry, flags);
+ if (unlikely(dentry->d_flags & DCACHE_OP_REVALIDATE))
+ return dentry->d_op->d_revalidate(dentry, flags);
+ else
+ return 1;
}
/**
if (nd->flags & LOOKUP_RCU) {
if (!(nd->flags & LOOKUP_ROOT))
nd->root.mnt = NULL;
- if (unlikely(unlazy_walk(nd, NULL, 0)))
+ if (unlikely(unlazy_walk(nd)))
return -ECHILD;
}
touch_atime(&last->link);
cond_resched();
} else if (atime_needs_update_rcu(&last->link, inode)) {
- if (unlikely(unlazy_walk(nd, NULL, 0)))
+ if (unlikely(unlazy_walk(nd)))
return ERR_PTR(-ECHILD);
touch_atime(&last->link);
}
if (nd->flags & LOOKUP_RCU) {
res = get(NULL, inode, &last->done);
if (res == ERR_PTR(-ECHILD)) {
- if (unlikely(unlazy_walk(nd, NULL, 0)))
+ if (unlikely(unlazy_walk(nd)))
return ERR_PTR(-ECHILD);
res = get(dentry, inode, &last->done);
}
struct dentry *dir,
unsigned int flags)
{
- struct dentry *dentry;
- int error;
-
- dentry = d_lookup(dir, name);
+ struct dentry *dentry = d_lookup(dir, name);
if (dentry) {
- if (dentry->d_flags & DCACHE_OP_REVALIDATE) {
- error = d_revalidate(dentry, flags);
- if (unlikely(error <= 0)) {
- if (!error)
- d_invalidate(dentry);
- dput(dentry);
- return ERR_PTR(error);
- }
+ int error = d_revalidate(dentry, flags);
+ if (unlikely(error <= 0)) {
+ if (!error)
+ d_invalidate(dentry);
+ dput(dentry);
+ return ERR_PTR(error);
}
}
return dentry;
bool negative;
dentry = __d_lookup_rcu(parent, &nd->last, &seq);
if (unlikely(!dentry)) {
- if (unlazy_walk(nd, NULL, 0))
+ if (unlazy_walk(nd))
return -ECHILD;
return 0;
}
return -ECHILD;
*seqp = seq;
- if (unlikely(dentry->d_flags & DCACHE_OP_REVALIDATE))
- status = d_revalidate(dentry, nd->flags);
- if (unlikely(status <= 0)) {
- if (unlazy_walk(nd, dentry, seq))
- return -ECHILD;
- if (status == -ECHILD)
- status = d_revalidate(dentry, nd->flags);
- } else {
+ status = d_revalidate(dentry, nd->flags);
+ if (likely(status > 0)) {
/*
* Note: do negative dentry check after revalidation in
* case that drops it.
path->dentry = dentry;
if (likely(__follow_mount_rcu(nd, path, inode, seqp)))
return 1;
- if (unlazy_walk(nd, dentry, seq))
- return -ECHILD;
}
+ if (unlazy_child(nd, dentry, seq))
+ return -ECHILD;
+ if (unlikely(status == -ECHILD))
+ /* we'd been told to redo it in non-rcu mode */
+ status = d_revalidate(dentry, nd->flags);
} else {
dentry = __d_lookup(parent, &nd->last);
if (unlikely(!dentry))
return 0;
- if (unlikely(dentry->d_flags & DCACHE_OP_REVALIDATE))
- status = d_revalidate(dentry, nd->flags);
+ status = d_revalidate(dentry, nd->flags);
}
if (unlikely(status <= 0)) {
if (!status)
if (IS_ERR(dentry))
goto out;
if (unlikely(!d_in_lookup(dentry))) {
- if ((dentry->d_flags & DCACHE_OP_REVALIDATE) &&
- !(flags & LOOKUP_NO_REVAL)) {
+ if (!(flags & LOOKUP_NO_REVAL)) {
int error = d_revalidate(dentry, flags);
if (unlikely(error <= 0)) {
if (!error) {
int err = inode_permission(nd->inode, MAY_EXEC|MAY_NOT_BLOCK);
if (err != -ECHILD)
return err;
- if (unlazy_walk(nd, NULL, 0))
+ if (unlazy_walk(nd))
return -ECHILD;
}
return inode_permission(nd->inode, MAY_EXEC);
error = nd_alloc_stack(nd);
if (unlikely(error)) {
if (error == -ECHILD) {
- if (unlikely(unlazy_link(nd, link, seq)))
- return -ECHILD;
- error = nd_alloc_stack(nd);
+ if (unlikely(!legitimize_path(nd, link, seq))) {
+ drop_links(nd);
+ nd->depth = 0;
+ nd->flags &= ~LOOKUP_RCU;
+ nd->path.mnt = NULL;
+ nd->path.dentry = NULL;
+ if (!(nd->flags & LOOKUP_ROOT))
+ nd->root.mnt = NULL;
+ rcu_read_unlock();
+ } else if (likely(unlazy_walk(nd)) == 0)
+ error = nd_alloc_stack(nd);
}
if (error) {
path_put(link);
}
if (unlikely(!d_can_lookup(nd->path.dentry))) {
if (nd->flags & LOOKUP_RCU) {
- if (unlazy_walk(nd, NULL, 0))
+ if (unlazy_walk(nd))
return -ECHILD;
}
return -ENOTDIR;
/* If we're in rcuwalk, drop out of it to handle last component */
if (nd->flags & LOOKUP_RCU) {
- if (unlazy_walk(nd, NULL, 0))
+ if (unlazy_walk(nd))
return -ECHILD;
}
if (d_in_lookup(dentry))
break;
- if (!(dentry->d_flags & DCACHE_OP_REVALIDATE))
- break;
-
error = d_revalidate(dentry, nd->flags);
if (likely(error > 0))
break;
return error;
}
+struct dentry *vfs_tmpfile(struct dentry *dentry, umode_t mode, int open_flag)
+{
+ static const struct qstr name = QSTR_INIT("/", 1);
+ struct dentry *child = NULL;
+ struct inode *dir = dentry->d_inode;
+ struct inode *inode;
+ int error;
+
+ /* we want directory to be writable */
+ error = inode_permission(dir, MAY_WRITE | MAY_EXEC);
+ if (error)
+ goto out_err;
+ error = -EOPNOTSUPP;
+ if (!dir->i_op->tmpfile)
+ goto out_err;
+ error = -ENOMEM;
+ child = d_alloc(dentry, &name);
+ if (unlikely(!child))
+ goto out_err;
+ error = dir->i_op->tmpfile(dir, child, mode);
+ if (error)
+ goto out_err;
+ error = -ENOENT;
+ inode = child->d_inode;
+ if (unlikely(!inode))
+ goto out_err;
+ if (!(open_flag & O_EXCL)) {
+ spin_lock(&inode->i_lock);
+ inode->i_state |= I_LINKABLE;
+ spin_unlock(&inode->i_lock);
+ }
+ return child;
+
+out_err:
+ dput(child);
+ return ERR_PTR(error);
+}
+EXPORT_SYMBOL(vfs_tmpfile);
+
static int do_tmpfile(struct nameidata *nd, unsigned flags,
const struct open_flags *op,
struct file *file, int *opened)
{
- static const struct qstr name = QSTR_INIT("/", 1);
struct dentry *child;
- struct inode *dir;
struct path path;
int error = path_lookupat(nd, flags | LOOKUP_DIRECTORY, &path);
if (unlikely(error))
error = mnt_want_write(path.mnt);
if (unlikely(error))
goto out;
- dir = path.dentry->d_inode;
- /* we want directory to be writable */
- error = inode_permission(dir, MAY_WRITE | MAY_EXEC);
- if (error)
+ child = vfs_tmpfile(path.dentry, op->mode, op->open_flag);
+ error = PTR_ERR(child);
+ if (unlikely(IS_ERR(child)))
goto out2;
- if (!dir->i_op->tmpfile) {
- error = -EOPNOTSUPP;
- goto out2;
- }
- child = d_alloc(path.dentry, &name);
- if (unlikely(!child)) {
- error = -ENOMEM;
- goto out2;
- }
dput(path.dentry);
path.dentry = child;
- error = dir->i_op->tmpfile(dir, child, op->mode);
- if (error)
- goto out2;
audit_inode(nd->name, child, 0);
/* Don't check for other permissions, the inode was just created */
error = may_open(&path, 0, op->open_flag);
if (error)
goto out2;
error = open_check_o_direct(file);
- if (error) {
+ if (error)
fput(file);
- } else if (!(op->open_flag & O_EXCL)) {
- struct inode *inode = file_inode(file);
- spin_lock(&inode->i_lock);
- inode->i_state |= I_LINKABLE;
- spin_unlock(&inode->i_lock);
- }
out2:
mnt_drop_write(path.mnt);
out:
#include <linux/user_namespace.h>
#include <linux/namei.h>
#include <linux/security.h>
+#include <linux/cred.h>
#include <linux/idr.h>
#include <linux/init.h> /* init_rootfs */
#include <linux/fs_struct.h> /* get_fs_root et.al. */
#include <linux/magic.h>
#include <linux/bootmem.h>
#include <linux/task_work.h>
+#include <linux/sched/task.h>
+
#include "pnode.h"
#include "internal.h"
#include <linux/vfs.h>
#include <linux/mount.h>
#include <linux/seq_file.h>
+#include <linux/sched/signal.h>
#include <linux/namei.h>
#include <net/sock.h>
#include <linux/highuid.h>
#include <linux/vmalloc.h>
#include <linux/sched.h>
+#include <linux/cred.h>
#include <linux/uaccess.h>
#include <linux/fcntl.h>
#include <linux/stat.h>
#include <linux/string.h>
+#include <linux/sched/signal.h>
#include <linux/uaccess.h>
#include <linux/in.h>
#include <linux/net.h>
return kernel_recvmsg(sock, &msg, &iov, 1, size, flags);
}
-static inline int do_send(struct socket *sock, struct kvec *vec, int count,
- int len, unsigned flags)
-{
- struct msghdr msg = { .msg_flags = flags };
- return kernel_sendmsg(sock, &msg, vec, count, len);
-}
-
static int _send(struct socket *sock, const void *buff, int len)
{
- struct kvec vec;
- vec.iov_base = (void *) buff;
- vec.iov_len = len;
- return do_send(sock, &vec, 1, len, 0);
+ struct msghdr msg = { .msg_flags = 0 };
+ struct kvec vec = {.iov_base = (void *)buff, .iov_len = len};
+ iov_iter_kvec(&msg.msg_iter, WRITE | ITER_KVEC, &vec, 1, len);
+ return sock_sendmsg(sock, &msg);
}
struct ncp_request_reply {
size_t datalen;
int result;
enum { RQ_DONE, RQ_INPROGRESS, RQ_QUEUED, RQ_IDLE, RQ_ABANDONED } status;
- struct kvec* tx_ciov;
- size_t tx_totallen;
- size_t tx_iovlen;
+ struct iov_iter from;
struct kvec tx_iov[3];
u_int16_t tx_type;
u_int32_t sign[6];
static int ncpdgram_send(struct socket *sock, struct ncp_request_reply *req)
{
- struct kvec vec[3];
- /* sock_sendmsg updates iov pointers for us :-( */
- memcpy(vec, req->tx_ciov, req->tx_iovlen * sizeof(vec[0]));
- return do_send(sock, vec, req->tx_iovlen,
- req->tx_totallen, MSG_DONTWAIT);
+ struct msghdr msg = { .msg_iter = req->from, .msg_flags = MSG_DONTWAIT };
+ return sock_sendmsg(sock, &msg);
}
static void __ncptcp_try_send(struct ncp_server *server)
{
struct ncp_request_reply *rq;
- struct kvec *iov;
- struct kvec iovc[3];
+ struct msghdr msg = { .msg_flags = MSG_NOSIGNAL | MSG_DONTWAIT };
int result;
rq = server->tx.creq;
if (!rq)
return;
- /* sock_sendmsg updates iov pointers for us :-( */
- memcpy(iovc, rq->tx_ciov, rq->tx_iovlen * sizeof(iov[0]));
- result = do_send(server->ncp_sock, iovc, rq->tx_iovlen,
- rq->tx_totallen, MSG_NOSIGNAL | MSG_DONTWAIT);
+ msg.msg_iter = rq->from;
+ result = sock_sendmsg(server->ncp_sock, &msg);
if (result == -EAGAIN)
return;
__ncp_abort_request(server, rq, result);
return;
}
- if (result >= rq->tx_totallen) {
+ if (!msg_data_left(&msg)) {
server->rcv.creq = rq;
server->tx.creq = NULL;
return;
}
- rq->tx_totallen -= result;
- iov = rq->tx_ciov;
- while (iov->iov_len <= result) {
- result -= iov->iov_len;
- iov++;
- rq->tx_iovlen--;
- }
- iov->iov_base += result;
- iov->iov_len -= result;
- rq->tx_ciov = iov;
+ rq->from = msg.msg_iter;
}
static inline void ncp_init_header(struct ncp_server *server, struct ncp_request_reply *req, struct ncp_request_header *h)
static void ncpdgram_start_request(struct ncp_server *server, struct ncp_request_reply *req)
{
- size_t signlen;
- struct ncp_request_header* h;
+ size_t signlen, len = req->tx_iov[1].iov_len;
+ struct ncp_request_header *h = req->tx_iov[1].iov_base;
- req->tx_ciov = req->tx_iov + 1;
-
- h = req->tx_iov[1].iov_base;
ncp_init_header(server, req, h);
- signlen = sign_packet(server, req->tx_iov[1].iov_base + sizeof(struct ncp_request_header) - 1,
- req->tx_iov[1].iov_len - sizeof(struct ncp_request_header) + 1,
- cpu_to_le32(req->tx_totallen), req->sign);
+ signlen = sign_packet(server,
+ req->tx_iov[1].iov_base + sizeof(struct ncp_request_header) - 1,
+ len - sizeof(struct ncp_request_header) + 1,
+ cpu_to_le32(len), req->sign);
if (signlen) {
- req->tx_ciov[1].iov_base = req->sign;
- req->tx_ciov[1].iov_len = signlen;
- req->tx_iovlen += 1;
- req->tx_totallen += signlen;
+ /* NCP over UDP appends signature */
+ req->tx_iov[2].iov_base = req->sign;
+ req->tx_iov[2].iov_len = signlen;
}
+ iov_iter_kvec(&req->from, WRITE | ITER_KVEC,
+ req->tx_iov + 1, signlen ? 2 : 1, len + signlen);
server->rcv.creq = req;
server->timeout_last = server->m.time_out;
server->timeout_retries = server->m.retry_count;
static void ncptcp_start_request(struct ncp_server *server, struct ncp_request_reply *req)
{
- size_t signlen;
- struct ncp_request_header* h;
+ size_t signlen, len = req->tx_iov[1].iov_len;
+ struct ncp_request_header *h = req->tx_iov[1].iov_base;
- req->tx_ciov = req->tx_iov;
- h = req->tx_iov[1].iov_base;
ncp_init_header(server, req, h);
signlen = sign_packet(server, req->tx_iov[1].iov_base + sizeof(struct ncp_request_header) - 1,
- req->tx_iov[1].iov_len - sizeof(struct ncp_request_header) + 1,
- cpu_to_be32(req->tx_totallen + 24), req->sign + 4) + 16;
+ len - sizeof(struct ncp_request_header) + 1,
+ cpu_to_be32(len + 24), req->sign + 4) + 16;
req->sign[0] = htonl(NCP_TCP_XMIT_MAGIC);
- req->sign[1] = htonl(req->tx_totallen + signlen);
+ req->sign[1] = htonl(len + signlen);
req->sign[2] = htonl(NCP_TCP_XMIT_VERSION);
req->sign[3] = htonl(req->datalen + 8);
+ /* NCP over TCP prepends signature */
req->tx_iov[0].iov_base = req->sign;
req->tx_iov[0].iov_len = signlen;
- req->tx_iovlen += 1;
- req->tx_totallen += signlen;
+ iov_iter_kvec(&req->from, WRITE | ITER_KVEC,
+ req->tx_iov, 2, len + signlen);
server->tx.creq = req;
__ncptcp_try_send(server);
static void info_server(struct ncp_server *server, unsigned int id, const void * data, size_t len)
{
if (server->info_sock) {
- struct kvec iov[2];
- __be32 hdr[2];
-
- hdr[0] = cpu_to_be32(len + 8);
- hdr[1] = cpu_to_be32(id);
-
- iov[0].iov_base = hdr;
- iov[0].iov_len = 8;
- iov[1].iov_base = (void *) data;
- iov[1].iov_len = len;
+ struct msghdr msg = { .msg_flags = MSG_NOSIGNAL };
+ __be32 hdr[2] = {cpu_to_be32(len + 8), cpu_to_be32(id)};
+ struct kvec iov[2] = {
+ {.iov_base = hdr, .iov_len = 8},
+ {.iov_base = (void *)data, .iov_len = len},
+ };
+
+ iov_iter_kvec(&msg.msg_iter, ITER_KVEC | WRITE,
+ iov, 2, len + 8);
- do_send(server->info_sock, iov, 2, len + 8, MSG_NOSIGNAL);
+ sock_sendmsg(server->info_sock, &msg);
}
}
req->datalen = max_reply_size;
req->tx_iov[1].iov_base = server->packet;
req->tx_iov[1].iov_len = size;
- req->tx_iovlen = 1;
- req->tx_totallen = size;
req->tx_type = *(u_int16_t*)server->packet;
result = ncp_add_request(server, req);
#include <linux/completion.h>
#include <linux/ip.h>
#include <linux/module.h>
+#include <linux/sched/signal.h>
#include <linux/sunrpc/svc.h>
#include <linux/sunrpc/svcsock.h>
#include <linux/nfs_fs.h>
#include <linux/module.h>
#include <linux/init.h>
-#include <linux/sched.h>
+#include <linux/sched/signal.h>
#include <linux/time.h>
#include <linux/kernel.h>
#include <linux/mm.h>
return false;
}
-int nfs_getattr(struct vfsmount *mnt, struct dentry *dentry, struct kstat *stat)
+int nfs_getattr(const struct path *path, struct kstat *stat,
+ u32 request_mask, unsigned int query_flags)
{
- struct inode *inode = d_inode(dentry);
+ struct inode *inode = d_inode(path->dentry);
int need_atime = NFS_I(inode)->cache_validity & NFS_INO_INVALID_ATIME;
int err = 0;
* - NFS never sets MS_NOATIME or MS_NODIRATIME so there is
* no point in checking those.
*/
- if ((mnt->mnt_flags & MNT_NOATIME) ||
- ((mnt->mnt_flags & MNT_NODIRATIME) && S_ISDIR(inode->i_mode)))
+ if ((path->mnt->mnt_flags & MNT_NOATIME) ||
+ ((path->mnt->mnt_flags & MNT_NODIRATIME) && S_ISDIR(inode->i_mode)))
need_atime = 0;
if (need_atime || nfs_need_revalidate_inode(inode)) {
struct nfs_server *server = NFS_SERVER(inode);
- nfs_readdirplus_parent_cache_miss(dentry);
+ nfs_readdirplus_parent_cache_miss(path->dentry);
err = __nfs_revalidate_inode(server, inode);
} else
- nfs_readdirplus_parent_cache_hit(dentry);
+ nfs_readdirplus_parent_cache_hit(path->dentry);
if (!err) {
generic_fillattr(inode, stat);
stat->ino = nfs_compat_user_ino64(NFS_FILEID(inode));
}
static int
-nfs_namespace_getattr(struct vfsmount *mnt, struct dentry *dentry, struct kstat *stat)
+nfs_namespace_getattr(const struct path *path, struct kstat *stat,
+ u32 request_mask, unsigned int query_flags)
{
- if (NFS_FH(d_inode(dentry))->size != 0)
- return nfs_getattr(mnt, dentry, stat);
- generic_fillattr(d_inode(dentry), stat);
+ if (NFS_FH(d_inode(path->dentry))->size != 0)
+ return nfs_getattr(path, stat, request_mask, query_flags);
+ generic_fillattr(d_inode(path->dentry), stat);
return 0;
}
if (ret < 0)
goto out_up;
- payload = user_key_payload(rkey);
+ payload = user_key_payload_rcu(rkey);
if (IS_ERR_OR_NULL(payload)) {
ret = PTR_ERR(payload);
goto out_up;
if (path.dentry != path.mnt->mnt_root)
break;
}
- err = vfs_getattr(&path, stat);
+ err = vfs_getattr(&path, stat, STATX_BASIC_STATS, AT_STATX_SYNC_AS_STAT);
path_put(&path);
return err;
}
goto out;
}
- err = vfs_getattr(&path, &stat);
+ err = vfs_getattr(&path, &stat, STATX_BASIC_STATS, AT_STATX_SYNC_AS_STAT);
if (err)
goto out_nfserr;
if ((bmval0 & (FATTR4_WORD0_FILES_AVAIL | FATTR4_WORD0_FILES_FREE |
* Copyright (C) 1995, 1996, 1997 Olaf Kirch <okir@monad.swb.de>
*/
-#include <linux/sched.h>
+#include <linux/sched/signal.h>
#include <linux/freezer.h>
#include <linux/module.h>
#include <linux/fs_struct.h>
{
struct path p = {.mnt = fh->fh_export->ex_path.mnt,
.dentry = fh->fh_dentry};
- return nfserrno(vfs_getattr(&p, stat));
+ return nfserrno(vfs_getattr(&p, stat, STATX_BASIC_STATS,
+ AT_STATX_SYNC_AS_STAT));
}
static inline int nfsd_create_is_exclusive(int createmode)
#include <linux/crc32.h>
#include <linux/pagevec.h>
#include <linux/slab.h>
+#include <linux/sched/signal.h>
+
#include "nilfs.h"
#include "btnode.h"
#include "page.h"
#include <linux/kernel.h> /* UINT_MAX */
#include <linux/mount.h>
#include <linux/sched.h>
+#include <linux/sched/user.h>
#include <linux/types.h>
#include <linux/wait.h>
#include <linux/types.h>
#include <linux/uaccess.h>
#include <linux/compat.h>
+#include <linux/sched/signal.h>
#include <asm/ioctls.h>
#include <linux/slab.h> /* kmem_* */
#include <linux/types.h>
#include <linux/sched.h>
+#include <linux/sched/user.h>
#include "inotify.h"
#include <linux/inotify.h>
#include <linux/kernel.h> /* roundup() */
#include <linux/namei.h> /* LOOKUP_FOLLOW */
-#include <linux/sched.h> /* struct user */
+#include <linux/sched/signal.h>
#include <linux/slab.h> /* struct kmem_cache */
#include <linux/syscalls.h>
#include <linux/types.h>
#include <linux/gfp.h>
#include <linux/pagemap.h>
#include <linux/pagevec.h>
-#include <linux/sched.h>
+#include <linux/sched/signal.h>
#include <linux/swap.h>
#include <linux/uio.h>
#include <linux/writeback.h>
#include <linux/swap.h>
#include <linux/quotaops.h>
#include <linux/blkdev.h>
+#include <linux/sched/signal.h>
#include <cluster/masklog.h>
*/
#include <linux/kernel.h>
+#include <linux/sched/mm.h>
#include <linux/jiffies.h>
#include <linux/slab.h>
#include <linux/idr.h>
#include <linux/delay.h>
#include <linux/err.h>
#include <linux/debugfs.h>
+#include <linux/sched/signal.h>
#include "cluster/heartbeat.h"
#include "cluster/nodemanager.h"
*/
#include <linux/signal.h>
+#include <linux/sched/signal.h>
#include <linux/module.h>
#include <linux/fs.h>
#include <linux/seq_file.h>
#include <linux/time.h>
#include <linux/quotaops.h>
+#include <linux/sched/signal.h>
#define MLOG_MASK_PREFIX ML_DLM_GLUE
#include <cluster/masklog.h>
return status;
}
-int ocfs2_getattr(struct vfsmount *mnt,
- struct dentry *dentry,
- struct kstat *stat)
+int ocfs2_getattr(const struct path *path, struct kstat *stat,
+ u32 request_mask, unsigned int flags)
{
- struct inode *inode = d_inode(dentry);
- struct super_block *sb = dentry->d_sb;
+ struct inode *inode = d_inode(path->dentry);
+ struct super_block *sb = path->dentry->d_sb;
struct ocfs2_super *osb = sb->s_fs_info;
int err;
- err = ocfs2_inode_revalidate(dentry);
+ err = ocfs2_inode_revalidate(path->dentry);
if (err) {
if (err != -ENOENT)
mlog_errno(err);
int ocfs2_extend_allocation(struct inode *inode, u32 logical_start,
u32 clusters_to_add, int mark_unwritten);
int ocfs2_setattr(struct dentry *dentry, struct iattr *attr);
-int ocfs2_getattr(struct vfsmount *mnt, struct dentry *dentry,
- struct kstat *stat);
+int ocfs2_getattr(const struct path *path, struct kstat *stat,
+ u32 request_mask, unsigned int flags);
int ocfs2_permission(struct inode *inode, int mask);
int ocfs2_should_update_atime(struct inode *inode,
#include <linux/seq_file.h>
#include <linux/quotaops.h>
#include <linux/cleancache.h>
+#include <linux/signal.h>
#define CREATE_TRACE_POINTS
#include "ocfs2_trace.h"
#include <linux/slab.h>
#include <linux/fs.h>
#include <linux/vfs.h>
+#include <linux/cred.h>
#include <linux/parser.h>
#include <linux/buffer_head.h>
#include <linux/vmalloc.h>
if (S_ISFIFO(inode->i_mode))
return -ESPIPE;
- /*
- * Let individual file system decide if it supports preallocation
- * for directories or not.
- */
- if (!S_ISREG(inode->i_mode) && !S_ISDIR(inode->i_mode) &&
- !S_ISBLK(inode->i_mode))
+ if (S_ISDIR(inode->i_mode))
+ return -EISDIR;
+
+ if (!S_ISREG(inode->i_mode) && !S_ISBLK(inode->i_mode))
return -ENODEV;
/* Check for wrap through zero too */
if (!file->f_op->fallocate)
return -EOPNOTSUPP;
- sb_start_write(inode->i_sb);
+ file_start_write(file);
ret = file->f_op->fallocate(file, mode, offset, len);
/*
if (ret == 0)
fsnotify_modify(file);
- sb_end_write(inode->i_sb);
+ file_end_write(file);
return ret;
}
EXPORT_SYMBOL_GPL(vfs_fallocate);
/*
* Obtain attributes of an object given a dentry
*/
-int orangefs_getattr(struct vfsmount *mnt,
- struct dentry *dentry,
- struct kstat *kstat)
+int orangefs_getattr(const struct path *path, struct kstat *stat,
+ u32 request_mask, unsigned int flags)
{
int ret = -ENOENT;
- struct inode *inode = dentry->d_inode;
+ struct inode *inode = path->dentry->d_inode;
struct orangefs_inode_s *orangefs_inode = NULL;
gossip_debug(GOSSIP_INODE_DEBUG,
"orangefs_getattr: called on %pd\n",
- dentry);
+ path->dentry);
ret = orangefs_inode_getattr(inode, 0, 0);
if (ret == 0) {
- generic_fillattr(inode, kstat);
+ generic_fillattr(inode, stat);
/* override block size reported to stat */
orangefs_inode = ORANGEFS_I(inode);
- kstat->blksize = orangefs_inode->blksize;
+ stat->blksize = orangefs_inode->blksize;
}
return ret;
}
#include <linux/uaccess.h>
#include <linux/atomic.h>
#include <linux/uio.h>
-#include <linux/sched.h>
+#include <linux/sched/signal.h>
#include <linux/mm.h>
#include <linux/wait.h>
#include <linux/dcache.h>
int orangefs_setattr(struct dentry *dentry, struct iattr *iattr);
-int orangefs_getattr(struct vfsmount *mnt,
- struct dentry *dentry,
- struct kstat *kstat);
+int orangefs_getattr(const struct path *path, struct kstat *stat,
+ u32 request_mask, unsigned int flags);
int orangefs_permission(struct inode *inode, int mask);
return &orangefs_inode->vfs_inode;
}
+static void orangefs_i_callback(struct rcu_head *head)
+{
+ struct inode *inode = container_of(head, struct inode, i_rcu);
+ struct orangefs_inode_s *orangefs_inode = ORANGEFS_I(inode);
+ kmem_cache_free(orangefs_inode_cache, orangefs_inode);
+}
+
static void orangefs_destroy_inode(struct inode *inode)
{
struct orangefs_inode_s *orangefs_inode = ORANGEFS_I(inode);
"%s: deallocated %p destroying inode %pU\n",
__func__, orangefs_inode, get_khandle_from_ino(inode));
- kmem_cache_free(orangefs_inode_cache, orangefs_inode);
+ call_rcu(&inode->i_rcu, orangefs_i_callback);
}
/*
#include <linux/xattr.h>
#include <linux/security.h>
#include <linux/uaccess.h>
-#include <linux/sched.h>
+#include <linux/sched/signal.h>
+#include <linux/cred.h>
#include <linux/namei.h>
#include <linux/fdtable.h>
#include <linux/ratelimit.h>
#include "overlayfs.h"
+#include "ovl_entry.h"
#define OVL_COPY_UP_CHUNK_SIZE (1 << 20)
static int ovl_copy_up_locked(struct dentry *workdir, struct dentry *upperdir,
struct dentry *dentry, struct path *lowerpath,
- struct kstat *stat, const char *link)
+ struct kstat *stat, const char *link,
+ struct kstat *pstat, bool tmpfile)
{
struct inode *wdir = workdir->d_inode;
struct inode *udir = upperdir->d_inode;
struct dentry *newdentry = NULL;
struct dentry *upper = NULL;
+ struct dentry *temp = NULL;
int err;
const struct cred *old_creds = NULL;
struct cred *new_creds = NULL;
.link = link
};
- newdentry = ovl_lookup_temp(workdir, dentry);
- err = PTR_ERR(newdentry);
- if (IS_ERR(newdentry))
- goto out;
-
upper = lookup_one_len(dentry->d_name.name, upperdir,
dentry->d_name.len);
err = PTR_ERR(upper);
if (IS_ERR(upper))
- goto out1;
+ goto out;
err = security_inode_copy_up(dentry, &new_creds);
if (err < 0)
- goto out2;
+ goto out1;
if (new_creds)
old_creds = override_creds(new_creds);
- err = ovl_create_real(wdir, newdentry, &cattr, NULL, true);
+ if (tmpfile)
+ temp = ovl_do_tmpfile(upperdir, stat->mode);
+ else
+ temp = ovl_lookup_temp(workdir, dentry);
+ err = PTR_ERR(temp);
+ if (IS_ERR(temp))
+ goto out1;
+
+ err = 0;
+ if (!tmpfile)
+ err = ovl_create_real(wdir, temp, &cattr, NULL, true);
if (new_creds) {
revert_creds(old_creds);
ovl_path_upper(dentry, &upperpath);
BUG_ON(upperpath.dentry != NULL);
- upperpath.dentry = newdentry;
+ upperpath.dentry = temp;
+
+ if (tmpfile) {
+ inode_unlock(udir);
+ err = ovl_copy_up_data(lowerpath, &upperpath,
+ stat->size);
+ inode_lock_nested(udir, I_MUTEX_PARENT);
+ } else {
+ err = ovl_copy_up_data(lowerpath, &upperpath,
+ stat->size);
+ }
- err = ovl_copy_up_data(lowerpath, &upperpath, stat->size);
if (err)
goto out_cleanup;
}
- err = ovl_copy_xattr(lowerpath->dentry, newdentry);
+ err = ovl_copy_xattr(lowerpath->dentry, temp);
if (err)
goto out_cleanup;
- inode_lock(newdentry->d_inode);
- err = ovl_set_attr(newdentry, stat);
- inode_unlock(newdentry->d_inode);
+ inode_lock(temp->d_inode);
+ err = ovl_set_attr(temp, stat);
+ inode_unlock(temp->d_inode);
if (err)
goto out_cleanup;
- err = ovl_do_rename(wdir, newdentry, udir, upper, 0);
+ if (tmpfile)
+ err = ovl_do_link(temp, udir, upper, true);
+ else
+ err = ovl_do_rename(wdir, temp, udir, upper, 0);
if (err)
goto out_cleanup;
+ newdentry = dget(tmpfile ? upper : temp);
ovl_dentry_update(dentry, newdentry);
ovl_inode_update(d_inode(dentry), d_inode(newdentry));
- newdentry = NULL;
+
+ /* Restore timestamps on parent (best effort) */
+ ovl_set_timestamps(upperdir, pstat);
out2:
- dput(upper);
+ dput(temp);
out1:
- dput(newdentry);
+ dput(upper);
out:
return err;
out_cleanup:
- ovl_cleanup(wdir, newdentry);
+ if (!tmpfile)
+ ovl_cleanup(wdir, temp);
goto out2;
}
struct dentry *lowerdentry = lowerpath->dentry;
struct dentry *upperdir;
const char *link = NULL;
+ struct ovl_fs *ofs = dentry->d_sb->s_fs_info;
if (WARN_ON(!workdir))
return -EROFS;
ovl_path_upper(parent, &parentpath);
upperdir = parentpath.dentry;
- err = vfs_getattr(&parentpath, &pstat);
+ err = vfs_getattr(&parentpath, &pstat,
+ STATX_ATIME | STATX_MTIME, AT_STATX_SYNC_AS_STAT);
if (err)
return err;
return PTR_ERR(link);
}
+ /* Should we copyup with O_TMPFILE or with workdir? */
+ if (S_ISREG(stat->mode) && ofs->tmpfile) {
+ err = ovl_copy_up_start(dentry);
+ /* err < 0: interrupted, err > 0: raced with another copy-up */
+ if (unlikely(err)) {
+ pr_debug("ovl_copy_up_start(%pd2) = %i\n", dentry, err);
+ if (err > 0)
+ err = 0;
+ goto out_done;
+ }
+
+ inode_lock_nested(upperdir->d_inode, I_MUTEX_PARENT);
+ err = ovl_copy_up_locked(workdir, upperdir, dentry, lowerpath,
+ stat, link, &pstat, true);
+ inode_unlock(upperdir->d_inode);
+ ovl_copy_up_end(dentry);
+ goto out_done;
+ }
+
err = -EIO;
if (lock_rename(workdir, upperdir) != NULL) {
pr_err("overlayfs: failed to lock workdir+upperdir\n");
}
err = ovl_copy_up_locked(workdir, upperdir, dentry, lowerpath,
- stat, link);
- if (!err) {
- /* Restore timestamps on parent (best effort) */
- ovl_set_timestamps(upperdir, &pstat);
- }
+ stat, link, &pstat, false);
out_unlock:
unlock_rename(workdir, upperdir);
+out_done:
do_delayed_call(&done);
return err;
}
ovl_path_lower(next, &lowerpath);
- err = vfs_getattr(&lowerpath, &stat);
+ err = vfs_getattr(&lowerpath, &stat,
+ STATX_BASIC_STATS, AT_STATX_SYNC_AS_STAT);
/* maybe truncate regular file. this has no effect on dirs */
if (flags & O_TRUNC)
stat.size = 0;
return err;
}
-static int ovl_dir_getattr(struct vfsmount *mnt, struct dentry *dentry,
- struct kstat *stat)
+static int ovl_dir_getattr(const struct path *path, struct kstat *stat,
+ u32 request_mask, unsigned int flags)
{
+ struct dentry *dentry = path->dentry;
int err;
enum ovl_path_type type;
struct path realpath;
type = ovl_path_real(dentry, &realpath);
old_cred = ovl_override_creds(dentry->d_sb);
- err = vfs_getattr(&realpath, stat);
+ err = vfs_getattr(&realpath, stat, request_mask, flags);
revert_creds(old_cred);
if (err)
return err;
goto out;
ovl_path_upper(dentry, &upperpath);
- err = vfs_getattr(&upperpath, &stat);
+ err = vfs_getattr(&upperpath, &stat,
+ STATX_BASIC_STATS, AT_STATX_SYNC_AS_STAT);
if (err)
goto out_unlock;
#include <linux/fs.h>
#include <linux/slab.h>
+#include <linux/cred.h>
#include <linux/xattr.h>
#include <linux/posix_acl.h>
#include "overlayfs.h"
return err;
}
-static int ovl_getattr(struct vfsmount *mnt, struct dentry *dentry,
- struct kstat *stat)
+static int ovl_getattr(const struct path *path, struct kstat *stat,
+ u32 request_mask, unsigned int flags)
{
+ struct dentry *dentry = path->dentry;
struct path realpath;
const struct cred *old_cred;
int err;
ovl_path_real(dentry, &realpath);
old_cred = ovl_override_creds(dentry->d_sb);
- err = vfs_getattr(&realpath, stat);
+ err = vfs_getattr(&realpath, stat, request_mask, flags);
revert_creds(old_cred);
return err;
}
*/
#include <linux/fs.h>
+#include <linux/cred.h>
#include <linux/namei.h>
#include <linux/xattr.h>
#include <linux/ratelimit.h>
return err;
}
+static inline struct dentry *ovl_do_tmpfile(struct dentry *dentry, umode_t mode)
+{
+ struct dentry *ret = vfs_tmpfile(dentry, mode, 0);
+ int err = IS_ERR(ret) ? PTR_ERR(ret) : 0;
+
+ pr_debug("tmpfile(%pd2, 0%o) = %i\n", dentry, mode, err);
+ return ret;
+}
+
static inline struct inode *ovl_inode_real(struct inode *inode, bool *is_upper)
{
unsigned long x = (unsigned long) READ_ONCE(inode->i_private);
u64 ovl_dentry_version_get(struct dentry *dentry);
bool ovl_is_whiteout(struct dentry *dentry);
struct file *ovl_path_open(struct path *path, int flags);
+int ovl_copy_up_start(struct dentry *dentry);
+void ovl_copy_up_end(struct dentry *dentry);
/* namei.c */
int ovl_path_next(int idx, struct dentry *dentry, struct path *path);
struct ovl_config config;
/* creds of process who forced instantiation of super block */
const struct cred *creator_cred;
+ bool tmpfile;
+ wait_queue_head_t copyup_wq;
};
/* private information held for every overlayfs dentry */
u64 version;
const char *redirect;
bool opaque;
+ bool copying;
};
struct rcu_head rcu;
};
* the Free Software Foundation.
*/
+#include <uapi/linux/magic.h>
#include <linux/fs.h>
#include <linux/namei.h>
#include <linux/xattr.h>
kfree(ufs);
}
+static int ovl_sync_fs(struct super_block *sb, int wait)
+{
+ struct ovl_fs *ufs = sb->s_fs_info;
+ struct super_block *upper_sb;
+ int ret;
+
+ if (!ufs->upper_mnt)
+ return 0;
+ upper_sb = ufs->upper_mnt->mnt_sb;
+ if (!upper_sb->s_op->sync_fs)
+ return 0;
+
+ /* real inodes have already been synced by sync_filesystem(ovl_sb) */
+ down_read(&upper_sb->s_umount);
+ ret = upper_sb->s_op->sync_fs(upper_sb, wait);
+ up_read(&upper_sb->s_umount);
+ return ret;
+}
+
/**
* ovl_statfs
* @sb: The overlayfs super block
static const struct super_operations ovl_super_operations = {
.put_super = ovl_put_super,
+ .sync_fs = ovl_sync_fs,
.statfs = ovl_statfs,
.show_options = ovl_show_options,
.remount_fs = ovl_remount,
unsigned int stacklen = 0;
unsigned int i;
bool remote = false;
+ struct cred *cred;
int err;
err = -ENOMEM;
if (!ufs)
goto out;
+ init_waitqueue_head(&ufs->copyup_wq);
ufs->config.redirect_dir = ovl_redirect_dir_def;
err = ovl_parse_opt((char *) data, &ufs->config);
if (err)
* creation of workdir in previous step.
*/
if (ufs->workdir) {
+ struct dentry *temp;
+
err = ovl_check_d_type_supported(&workpath);
if (err < 0)
goto out_put_workdir;
*/
if (!err)
pr_warn("overlayfs: upper fs needs to support d_type.\n");
+
+ /* Check if upper/work fs supports O_TMPFILE */
+ temp = ovl_do_tmpfile(ufs->workdir, S_IFREG | 0);
+ ufs->tmpfile = !IS_ERR(temp);
+ if (ufs->tmpfile)
+ dput(temp);
+ else
+ pr_warn("overlayfs: upper fs does not support tmpfile.\n");
}
}
else
sb->s_d_op = &ovl_dentry_operations;
- ufs->creator_cred = prepare_creds();
- if (!ufs->creator_cred)
+ ufs->creator_cred = cred = prepare_creds();
+ if (!cred)
goto out_put_lower_mnt;
+ /* Never override disk quota limits or use reserved space */
+ cap_lower(cred->cap_effective, CAP_SYS_RESOURCE);
+
err = -ENOMEM;
oe = ovl_alloc_entry(numlower);
if (!oe)
#include <linux/fs.h>
#include <linux/mount.h>
#include <linux/slab.h>
+#include <linux/cred.h>
#include <linux/xattr.h>
#include "overlayfs.h"
#include "ovl_entry.h"
{
return dentry_open(path, flags | O_NOATIME, current_cred());
}
+
+int ovl_copy_up_start(struct dentry *dentry)
+{
+ struct ovl_fs *ofs = dentry->d_sb->s_fs_info;
+ struct ovl_entry *oe = dentry->d_fsdata;
+ int err;
+
+ spin_lock(&ofs->copyup_wq.lock);
+ err = wait_event_interruptible_locked(ofs->copyup_wq, !oe->copying);
+ if (!err) {
+ if (oe->__upperdentry)
+ err = 1; /* Already copied up */
+ else
+ oe->copying = true;
+ }
+ spin_unlock(&ofs->copyup_wq.lock);
+
+ return err;
+}
+
+void ovl_copy_up_end(struct dentry *dentry)
+{
+ struct ovl_fs *ofs = dentry->d_sb->s_fs_info;
+ struct ovl_entry *oe = dentry->d_fsdata;
+
+ spin_lock(&ofs->copyup_wq.lock);
+ oe->copying = false;
+ wake_up_locked(&ofs->copyup_wq);
+ spin_unlock(&ofs->copyup_wq.lock);
+}
#include <linux/atomic.h>
#include <linux/fs.h>
#include <linux/sched.h>
+#include <linux/cred.h>
#include <linux/posix_acl.h>
#include <linux/posix_acl_xattr.h>
#include <linux/xattr.h>
#include <linux/tty.h>
#include <linux/string.h>
#include <linux/mman.h>
+#include <linux/sched/mm.h>
+#include <linux/sched/numa_balancing.h>
+#include <linux/sched/task.h>
+#include <linux/sched/cputime.h>
#include <linux/proc_fs.h>
#include <linux/ioport.h>
#include <linux/uaccess.h>
#include <linux/user_namespace.h>
#include <linux/fs_struct.h>
#include <linux/slab.h>
+#include <linux/sched/autogroup.h>
+#include <linux/sched/mm.h>
+#include <linux/sched/coredump.h>
+#include <linux/sched/debug.h>
+#include <linux/sched/stat.h>
#include <linux/flex_array.h>
#include <linux/posix-timers.h>
#ifdef CONFIG_HARDWALL
return NULL;
}
-int pid_getattr(struct vfsmount *mnt, struct dentry *dentry, struct kstat *stat)
+int pid_getattr(const struct path *path, struct kstat *stat,
+ u32 request_mask, unsigned int query_flags)
{
- struct inode *inode = d_inode(dentry);
+ struct inode *inode = d_inode(path->dentry);
struct task_struct *task;
- struct pid_namespace *pid = dentry->d_sb->s_fs_info;
+ struct pid_namespace *pid = path->dentry->d_sb->s_fs_info;
generic_fillattr(inode, stat);
return 0;
}
-static int proc_task_getattr(struct vfsmount *mnt, struct dentry *dentry, struct kstat *stat)
+static int proc_task_getattr(const struct path *path, struct kstat *stat,
+ u32 request_mask, unsigned int query_flags)
{
- struct inode *inode = d_inode(dentry);
+ struct inode *inode = d_inode(path->dentry);
struct task_struct *p = get_proc_task(inode);
generic_fillattr(inode, stat);
-#include <linux/sched.h>
+#include <linux/sched/signal.h>
#include <linux/errno.h>
#include <linux/dcache.h>
#include <linux/path.h>
return 0;
}
-static int proc_getattr(struct vfsmount *mnt, struct dentry *dentry,
- struct kstat *stat)
+static int proc_getattr(const struct path *path, struct kstat *stat,
+ u32 request_mask, unsigned int query_flags)
{
- struct inode *inode = d_inode(dentry);
+ struct inode *inode = d_inode(path->dentry);
struct proc_dir_entry *de = PDE(inode);
if (de && de->nlink)
set_nlink(inode, de->nlink);
#include <linux/spinlock.h>
#include <linux/atomic.h>
#include <linux/binfmts.h>
+#include <linux/sched/coredump.h>
+#include <linux/sched/task.h>
struct ctl_table_header;
struct mempolicy;
* base.c
*/
extern const struct dentry_operations pid_dentry_operations;
-extern int pid_getattr(struct vfsmount *, struct dentry *, struct kstat *);
+extern int pid_getattr(const struct path *, struct kstat *, u32, unsigned int);
extern int proc_setattr(struct dentry *, struct iattr *);
extern struct inode *proc_pid_make_inode(struct super_block *, struct task_struct *, umode_t);
extern int pid_revalidate(struct dentry *, unsigned int);
#include <linux/list.h>
#include <linux/ioport.h>
#include <linux/memory.h>
+#include <linux/sched/task.h>
#include <asm/sections.h>
#include "internal.h"
#include <linux/pid_namespace.h>
#include <linux/proc_fs.h>
#include <linux/sched.h>
+#include <linux/sched/loadavg.h>
+#include <linux/sched/stat.h>
#include <linux/seq_file.h>
#include <linux/seqlock.h>
#include <linux/time.h>
#include <linux/slab.h>
#include <linux/init.h>
#include <linux/sched.h>
+#include <linux/sched/task.h>
#include <linux/module.h>
#include <linux/bitops.h>
#include <linux/mount.h>
return de;
}
-static int proc_tgid_net_getattr(struct vfsmount *mnt, struct dentry *dentry,
- struct kstat *stat)
+static int proc_tgid_net_getattr(const struct path *path, struct kstat *stat,
+ u32 request_mask, unsigned int query_flags)
{
- struct inode *inode = d_inode(dentry);
+ struct inode *inode = d_inode(path->dentry);
struct net *net;
net = get_proc_task_net(inode);
#include <linux/printk.h>
#include <linux/security.h>
#include <linux/sched.h>
+#include <linux/cred.h>
#include <linux/namei.h>
#include <linux/mm.h>
#include <linux/module.h>
return 0;
}
-static int proc_sys_getattr(struct vfsmount *mnt, struct dentry *dentry, struct kstat *stat)
+static int proc_sys_getattr(const struct path *path, struct kstat *stat,
+ u32 request_mask, unsigned int query_flags)
{
- struct inode *inode = d_inode(dentry);
+ struct inode *inode = d_inode(path->dentry);
struct ctl_table_header *head = grab_header(inode);
struct ctl_table *table = PROC_I(inode)->sysctl_entry;
#include <linux/stat.h>
#include <linux/init.h>
#include <linux/sched.h>
+#include <linux/sched/stat.h>
#include <linux/module.h>
#include <linux/bitops.h>
#include <linux/user_namespace.h>
#include <linux/mount.h>
#include <linux/pid_namespace.h>
#include <linux/parser.h>
+#include <linux/cred.h>
#include "internal.h"
proc_sys_init();
}
-static int proc_root_getattr(struct vfsmount *mnt, struct dentry *dentry, struct kstat *stat
-)
+static int proc_root_getattr(const struct path *path, struct kstat *stat,
+ u32 request_mask, unsigned int query_flags)
{
- generic_fillattr(d_inode(dentry), stat);
+ generic_fillattr(d_inode(path->dentry), stat);
stat->nlink = proc_root.nlink + nr_processes();
return 0;
}
#include <linux/kernel_stat.h>
#include <linux/proc_fs.h>
#include <linux/sched.h>
+#include <linux/sched/stat.h>
#include <linux/seq_file.h>
#include <linux/slab.h>
#include <linux/time.h>
#include <linux/irqnr.h>
-#include <linux/cputime.h>
+#include <linux/sched/cputime.h>
#include <linux/tick.h>
#ifndef arch_irq_stat_cpu
#include <linux/mempolicy.h>
#include <linux/rmap.h>
#include <linux/swap.h>
+#include <linux/sched/mm.h>
#include <linux/swapops.h>
#include <linux/mmu_notifier.h>
#include <linux/page_idle.h>
#include <linux/ptrace.h>
#include <linux/slab.h>
#include <linux/seq_file.h>
+#include <linux/sched/mm.h>
+
#include "internal.h"
/*
#include <linux/nsproxy.h>
#include <linux/security.h>
#include <linux/fs_struct.h>
+#include <linux/sched/task.h>
+
#include "proc/internal.h" /* only for get_proc_task() in ->open() */
#include "pnode.h"
#include <linux/proc_fs.h>
#include <linux/security.h>
#include <linux/sched.h>
+#include <linux/cred.h>
#include <linux/kmod.h>
#include <linux/namei.h>
#include <linux/capability.h>
* Copyright (C) 1991, 1992 Linus Torvalds
*/
-#include <linux/slab.h>
+#include <linux/slab.h>
#include <linux/stat.h>
+#include <linux/sched/xacct.h>
#include <linux/fcntl.h>
#include <linux/file.h>
#include <linux/uio.h>
#include <linux/uaccess.h>
#include <asm/unistd.h>
-typedef ssize_t (*io_fn_t)(struct file *, char __user *, size_t, loff_t *);
-typedef ssize_t (*iter_fn_t)(struct kiocb *, struct iov_iter *);
-
const struct file_operations generic_ro_fops = {
.llseek = generic_file_llseek,
.read_iter = generic_file_read_iter,
kiocb.ki_pos = *ppos;
iter->type |= READ;
- ret = file->f_op->read_iter(&kiocb, iter);
+ ret = call_read_iter(file, &kiocb, iter);
BUG_ON(ret == -EIOCBQUEUED);
if (ret > 0)
*ppos = kiocb.ki_pos;
kiocb.ki_pos = *ppos;
iter->type |= WRITE;
- ret = file->f_op->write_iter(&kiocb, iter);
+ ret = call_write_iter(file, &kiocb, iter);
BUG_ON(ret == -EIOCBQUEUED);
if (ret > 0)
*ppos = kiocb.ki_pos;
kiocb.ki_pos = *ppos;
iov_iter_init(&iter, READ, &iov, 1, len);
- ret = filp->f_op->read_iter(&kiocb, &iter);
+ ret = call_read_iter(filp, &kiocb, &iter);
BUG_ON(ret == -EIOCBQUEUED);
*ppos = kiocb.ki_pos;
return ret;
kiocb.ki_pos = *ppos;
iov_iter_init(&iter, WRITE, &iov, 1, len);
- ret = filp->f_op->write_iter(&kiocb, &iter);
+ ret = call_write_iter(filp, &kiocb, &iter);
BUG_ON(ret == -EIOCBQUEUED);
if (ret > 0)
*ppos = kiocb.ki_pos;
EXPORT_SYMBOL(iov_shorten);
static ssize_t do_iter_readv_writev(struct file *filp, struct iov_iter *iter,
- loff_t *ppos, iter_fn_t fn, int flags)
+ loff_t *ppos, int type, int flags)
{
struct kiocb kiocb;
ssize_t ret;
kiocb.ki_flags |= (IOCB_DSYNC | IOCB_SYNC);
kiocb.ki_pos = *ppos;
- ret = fn(&kiocb, iter);
+ if (type == READ)
+ ret = call_read_iter(filp, &kiocb, iter);
+ else
+ ret = call_write_iter(filp, &kiocb, iter);
BUG_ON(ret == -EIOCBQUEUED);
*ppos = kiocb.ki_pos;
return ret;
/* Do it by hand, with file-ops */
static ssize_t do_loop_readv_writev(struct file *filp, struct iov_iter *iter,
- loff_t *ppos, io_fn_t fn, int flags)
+ loff_t *ppos, int type, int flags)
{
ssize_t ret = 0;
struct iovec iovec = iov_iter_iovec(iter);
ssize_t nr;
- nr = fn(filp, iovec.iov_base, iovec.iov_len, ppos);
+ if (type == READ) {
+ nr = filp->f_op->read(filp, iovec.iov_base,
+ iovec.iov_len, ppos);
+ } else {
+ nr = filp->f_op->write(filp, iovec.iov_base,
+ iovec.iov_len, ppos);
+ }
if (nr < 0) {
if (!ret)
return ret;
}
-static ssize_t do_readv_writev(int type, struct file *file,
- const struct iovec __user * uvector,
- unsigned long nr_segs, loff_t *pos,
- int flags)
+static ssize_t __do_readv_writev(int type, struct file *file,
+ struct iov_iter *iter, loff_t *pos, int flags)
{
size_t tot_len;
- struct iovec iovstack[UIO_FASTIOV];
- struct iovec *iov = iovstack;
- struct iov_iter iter;
- ssize_t ret;
- io_fn_t fn;
- iter_fn_t iter_fn;
-
- ret = import_iovec(type, uvector, nr_segs,
- ARRAY_SIZE(iovstack), &iov, &iter);
- if (ret < 0)
- return ret;
+ ssize_t ret = 0;
- tot_len = iov_iter_count(&iter);
+ tot_len = iov_iter_count(iter);
if (!tot_len)
goto out;
ret = rw_verify_area(type, file, pos, tot_len);
if (ret < 0)
goto out;
- if (type == READ) {
- fn = file->f_op->read;
- iter_fn = file->f_op->read_iter;
- } else {
- fn = (io_fn_t)file->f_op->write;
- iter_fn = file->f_op->write_iter;
+ if (type != READ)
file_start_write(file);
- }
- if (iter_fn)
- ret = do_iter_readv_writev(file, &iter, pos, iter_fn, flags);
+ if ((type == READ && file->f_op->read_iter) ||
+ (type == WRITE && file->f_op->write_iter))
+ ret = do_iter_readv_writev(file, iter, pos, type, flags);
else
- ret = do_loop_readv_writev(file, &iter, pos, fn, flags);
+ ret = do_loop_readv_writev(file, iter, pos, type, flags);
if (type != READ)
file_end_write(file);
out:
- kfree(iov);
if ((ret + (type == READ)) > 0) {
if (type == READ)
fsnotify_access(file);
return ret;
}
+static ssize_t do_readv_writev(int type, struct file *file,
+ const struct iovec __user *uvector,
+ unsigned long nr_segs, loff_t *pos,
+ int flags)
+{
+ struct iovec iovstack[UIO_FASTIOV];
+ struct iovec *iov = iovstack;
+ struct iov_iter iter;
+ ssize_t ret;
+
+ ret = import_iovec(type, uvector, nr_segs,
+ ARRAY_SIZE(iovstack), &iov, &iter);
+ if (ret < 0)
+ return ret;
+
+ ret = __do_readv_writev(type, file, &iter, pos, flags);
+ kfree(iov);
+
+ return ret;
+}
+
ssize_t vfs_readv(struct file *file, const struct iovec __user *vec,
unsigned long vlen, loff_t *pos, int flags)
{
unsigned long nr_segs, loff_t *pos,
int flags)
{
- compat_ssize_t tot_len;
struct iovec iovstack[UIO_FASTIOV];
struct iovec *iov = iovstack;
struct iov_iter iter;
ssize_t ret;
- io_fn_t fn;
- iter_fn_t iter_fn;
ret = compat_import_iovec(type, uvector, nr_segs,
UIO_FASTIOV, &iov, &iter);
if (ret < 0)
return ret;
- tot_len = iov_iter_count(&iter);
- if (!tot_len)
- goto out;
- ret = rw_verify_area(type, file, pos, tot_len);
- if (ret < 0)
- goto out;
-
- if (type == READ) {
- fn = file->f_op->read;
- iter_fn = file->f_op->read_iter;
- } else {
- fn = (io_fn_t)file->f_op->write;
- iter_fn = file->f_op->write_iter;
- file_start_write(file);
- }
-
- if (iter_fn)
- ret = do_iter_readv_writev(file, &iter, pos, iter_fn, flags);
- else
- ret = do_loop_readv_writev(file, &iter, pos, fn, flags);
-
- if (type != READ)
- file_end_write(file);
-
-out:
+ ret = __do_readv_writev(type, file, &iter, pos, flags);
kfree(iov);
- if ((ret + (type == READ)) > 0) {
- if (type == READ)
- fsnotify_access(file);
- else
- fsnotify_modify(file);
- }
+
return ret;
}
if (flags != 0)
return -EINVAL;
+ if (S_ISDIR(inode_in->i_mode) || S_ISDIR(inode_out->i_mode))
+ return -EISDIR;
+ if (!S_ISREG(inode_in->i_mode) || !S_ISREG(inode_out->i_mode))
+ return -EINVAL;
+
ret = rw_verify_area(READ, file_in, &pos_in, len);
if (unlikely(ret))
return ret;
if (len == 0)
return 0;
- sb_start_write(inode_out->i_sb);
+ file_start_write(file_out);
/*
* Try cloning first, this is supported by more file systems, and
inc_syscr(current);
inc_syscw(current);
- sb_end_write(inode_out->i_sb);
+ file_end_write(file_out);
return ret;
}
*/
#include <linux/kernel.h>
-#include <linux/sched.h>
+#include <linux/sched/signal.h>
+#include <linux/sched/rt.h>
#include <linux/syscalls.h>
#include <linux/export.h>
#include <linux/slab.h>
#include <linux/fs.h>
#include <linux/rcupdate.h>
#include <linux/hrtimer.h>
-#include <linux/sched/rt.h>
#include <linux/freezer.h>
#include <net/busy_poll.h>
#include <linux/vmalloc.h>
#include <linux/gfp.h>
#include <linux/socket.h>
#include <linux/compat.h>
+#include <linux/sched/signal.h>
+
#include "internal.h"
/*
idx = to.idx;
init_sync_kiocb(&kiocb, in);
kiocb.ki_pos = *ppos;
- ret = in->f_op->read_iter(&kiocb, &to);
+ ret = call_read_iter(in, &kiocb, &to);
if (ret > 0) {
*ppos = kiocb.ki_pos;
file_accessed(in);
#include <linux/fs.h>
#include <linux/namei.h>
#include <linux/security.h>
+#include <linux/cred.h>
#include <linux/syscalls.h>
#include <linux/pagemap.h>
#include <linux/uaccess.h>
#include <asm/unistd.h>
+/**
+ * generic_fillattr - Fill in the basic attributes from the inode struct
+ * @inode: Inode to use as the source
+ * @stat: Where to fill in the attributes
+ *
+ * Fill in the basic attributes in the kstat structure from data that's to be
+ * found on the VFS inode structure. This is the default if no getattr inode
+ * operation is supplied.
+ */
void generic_fillattr(struct inode *inode, struct kstat *stat)
{
stat->dev = inode->i_sb->s_dev;
stat->ctime = inode->i_ctime;
stat->blksize = i_blocksize(inode);
stat->blocks = inode->i_blocks;
-}
+ if (IS_NOATIME(inode))
+ stat->result_mask &= ~STATX_ATIME;
+ if (IS_AUTOMOUNT(inode))
+ stat->attributes |= STATX_ATTR_AUTOMOUNT;
+}
EXPORT_SYMBOL(generic_fillattr);
/**
* vfs_getattr_nosec - getattr without security checks
* @path: file to get attributes from
* @stat: structure to return attributes in
+ * @request_mask: STATX_xxx flags indicating what the caller wants
+ * @query_flags: Query mode (KSTAT_QUERY_FLAGS)
*
* Get attributes without calling security_inode_getattr.
*
* Currently the only caller other than vfs_getattr is internal to the
- * filehandle lookup code, which uses only the inode number and returns
- * no attributes to any user. Any other code probably wants
- * vfs_getattr.
+ * filehandle lookup code, which uses only the inode number and returns no
+ * attributes to any user. Any other code probably wants vfs_getattr.
*/
-int vfs_getattr_nosec(struct path *path, struct kstat *stat)
+int vfs_getattr_nosec(const struct path *path, struct kstat *stat,
+ u32 request_mask, unsigned int query_flags)
{
struct inode *inode = d_backing_inode(path->dentry);
+ memset(stat, 0, sizeof(*stat));
+ stat->result_mask |= STATX_BASIC_STATS;
+ request_mask &= STATX_ALL;
+ query_flags &= KSTAT_QUERY_FLAGS;
if (inode->i_op->getattr)
- return inode->i_op->getattr(path->mnt, path->dentry, stat);
+ return inode->i_op->getattr(path, stat, request_mask,
+ query_flags);
generic_fillattr(inode, stat);
return 0;
}
-
EXPORT_SYMBOL(vfs_getattr_nosec);
-int vfs_getattr(struct path *path, struct kstat *stat)
+/*
+ * vfs_getattr - Get the enhanced basic attributes of a file
+ * @path: The file of interest
+ * @stat: Where to return the statistics
+ * @request_mask: STATX_xxx flags indicating what the caller wants
+ * @query_flags: Query mode (KSTAT_QUERY_FLAGS)
+ *
+ * Ask the filesystem for a file's attributes. The caller must indicate in
+ * request_mask and query_flags to indicate what they want.
+ *
+ * If the file is remote, the filesystem can be forced to update the attributes
+ * from the backing store by passing AT_STATX_FORCE_SYNC in query_flags or can
+ * suppress the update by passing AT_STATX_DONT_SYNC.
+ *
+ * Bits must have been set in request_mask to indicate which attributes the
+ * caller wants retrieving. Any such attribute not requested may be returned
+ * anyway, but the value may be approximate, and, if remote, may not have been
+ * synchronised with the server.
+ *
+ * 0 will be returned on success, and a -ve error code if unsuccessful.
+ */
+int vfs_getattr(const struct path *path, struct kstat *stat,
+ u32 request_mask, unsigned int query_flags)
{
int retval;
retval = security_inode_getattr(path);
if (retval)
return retval;
- return vfs_getattr_nosec(path, stat);
+ return vfs_getattr_nosec(path, stat, request_mask, query_flags);
}
-
EXPORT_SYMBOL(vfs_getattr);
-int vfs_fstat(unsigned int fd, struct kstat *stat)
+/**
+ * vfs_statx_fd - Get the enhanced basic attributes by file descriptor
+ * @fd: The file descriptor referring to the file of interest
+ * @stat: The result structure to fill in.
+ * @request_mask: STATX_xxx flags indicating what the caller wants
+ * @query_flags: Query mode (KSTAT_QUERY_FLAGS)
+ *
+ * This function is a wrapper around vfs_getattr(). The main difference is
+ * that it uses a file descriptor to determine the file location.
+ *
+ * 0 will be returned on success, and a -ve error code if unsuccessful.
+ */
+int vfs_statx_fd(unsigned int fd, struct kstat *stat,
+ u32 request_mask, unsigned int query_flags)
{
struct fd f = fdget_raw(fd);
int error = -EBADF;
if (f.file) {
- error = vfs_getattr(&f.file->f_path, stat);
+ error = vfs_getattr(&f.file->f_path, stat,
+ request_mask, query_flags);
fdput(f);
}
return error;
}
-EXPORT_SYMBOL(vfs_fstat);
+EXPORT_SYMBOL(vfs_statx_fd);
-int vfs_fstatat(int dfd, const char __user *filename, struct kstat *stat,
- int flag)
+/**
+ * vfs_statx - Get basic and extra attributes by filename
+ * @dfd: A file descriptor representing the base dir for a relative filename
+ * @filename: The name of the file of interest
+ * @flags: Flags to control the query
+ * @stat: The result structure to fill in.
+ * @request_mask: STATX_xxx flags indicating what the caller wants
+ *
+ * This function is a wrapper around vfs_getattr(). The main difference is
+ * that it uses a filename and base directory to determine the file location.
+ * Additionally, the use of AT_SYMLINK_NOFOLLOW in flags will prevent a symlink
+ * at the given name from being referenced.
+ *
+ * The caller must have preset stat->request_mask as for vfs_getattr(). The
+ * flags are also used to load up stat->query_flags.
+ *
+ * 0 will be returned on success, and a -ve error code if unsuccessful.
+ */
+int vfs_statx(int dfd, const char __user *filename, int flags,
+ struct kstat *stat, u32 request_mask)
{
struct path path;
int error = -EINVAL;
- unsigned int lookup_flags = 0;
+ unsigned int lookup_flags = LOOKUP_FOLLOW | LOOKUP_AUTOMOUNT;
- if ((flag & ~(AT_SYMLINK_NOFOLLOW | AT_NO_AUTOMOUNT |
- AT_EMPTY_PATH)) != 0)
- goto out;
+ if ((flags & ~(AT_SYMLINK_NOFOLLOW | AT_NO_AUTOMOUNT |
+ AT_EMPTY_PATH | KSTAT_QUERY_FLAGS)) != 0)
+ return -EINVAL;
- if (!(flag & AT_SYMLINK_NOFOLLOW))
- lookup_flags |= LOOKUP_FOLLOW;
- if (flag & AT_EMPTY_PATH)
+ if (flags & AT_SYMLINK_NOFOLLOW)
+ lookup_flags &= ~LOOKUP_FOLLOW;
+ if (flags & AT_NO_AUTOMOUNT)
+ lookup_flags &= ~LOOKUP_AUTOMOUNT;
+ if (flags & AT_EMPTY_PATH)
lookup_flags |= LOOKUP_EMPTY;
+
retry:
error = user_path_at(dfd, filename, lookup_flags, &path);
if (error)
goto out;
- error = vfs_getattr(&path, stat);
+ error = vfs_getattr(&path, stat, request_mask, flags);
path_put(&path);
if (retry_estale(error, lookup_flags)) {
lookup_flags |= LOOKUP_REVAL;
out:
return error;
}
-EXPORT_SYMBOL(vfs_fstatat);
-
-int vfs_stat(const char __user *name, struct kstat *stat)
-{
- return vfs_fstatat(AT_FDCWD, name, stat, 0);
-}
-EXPORT_SYMBOL(vfs_stat);
-
-int vfs_lstat(const char __user *name, struct kstat *stat)
-{
- return vfs_fstatat(AT_FDCWD, name, stat, AT_SYMLINK_NOFOLLOW);
-}
-EXPORT_SYMBOL(vfs_lstat);
+EXPORT_SYMBOL(vfs_statx);
#ifdef __ARCH_WANT_OLD_STAT
{
static int warncount = 5;
struct __old_kernel_stat tmp;
-
+
if (warncount > 0) {
warncount--;
printk(KERN_WARNING "VFS: Warning: %s using old stat() call. Recompile your binary.\n",
#if BITS_PER_LONG == 32
if (stat->size > MAX_NON_LFS)
return -EOVERFLOW;
-#endif
+#endif
tmp.st_size = stat->size;
tmp.st_atime = stat->atime.tv_sec;
tmp.st_mtime = stat->mtime.tv_sec;
}
#endif /* __ARCH_WANT_STAT64 || __ARCH_WANT_COMPAT_STAT64 */
+static inline int __put_timestamp(struct timespec *kts,
+ struct statx_timestamp __user *uts)
+{
+ return (__put_user(kts->tv_sec, &uts->tv_sec ) ||
+ __put_user(kts->tv_nsec, &uts->tv_nsec ) ||
+ __put_user(0, &uts->__reserved ));
+}
+
+/*
+ * Set the statx results.
+ */
+static long statx_set_result(struct kstat *stat, struct statx __user *buffer)
+{
+ uid_t uid = from_kuid_munged(current_user_ns(), stat->uid);
+ gid_t gid = from_kgid_munged(current_user_ns(), stat->gid);
+
+ if (__put_user(stat->result_mask, &buffer->stx_mask ) ||
+ __put_user(stat->mode, &buffer->stx_mode ) ||
+ __clear_user(&buffer->__spare0, sizeof(buffer->__spare0)) ||
+ __put_user(stat->nlink, &buffer->stx_nlink ) ||
+ __put_user(uid, &buffer->stx_uid ) ||
+ __put_user(gid, &buffer->stx_gid ) ||
+ __put_user(stat->attributes, &buffer->stx_attributes ) ||
+ __put_user(stat->blksize, &buffer->stx_blksize ) ||
+ __put_user(MAJOR(stat->rdev), &buffer->stx_rdev_major ) ||
+ __put_user(MINOR(stat->rdev), &buffer->stx_rdev_minor ) ||
+ __put_user(MAJOR(stat->dev), &buffer->stx_dev_major ) ||
+ __put_user(MINOR(stat->dev), &buffer->stx_dev_minor ) ||
+ __put_timestamp(&stat->atime, &buffer->stx_atime ) ||
+ __put_timestamp(&stat->btime, &buffer->stx_btime ) ||
+ __put_timestamp(&stat->ctime, &buffer->stx_ctime ) ||
+ __put_timestamp(&stat->mtime, &buffer->stx_mtime ) ||
+ __put_user(stat->ino, &buffer->stx_ino ) ||
+ __put_user(stat->size, &buffer->stx_size ) ||
+ __put_user(stat->blocks, &buffer->stx_blocks ) ||
+ __clear_user(&buffer->__spare1, sizeof(buffer->__spare1)) ||
+ __clear_user(&buffer->__spare2, sizeof(buffer->__spare2)))
+ return -EFAULT;
+
+ return 0;
+}
+
+/**
+ * sys_statx - System call to get enhanced stats
+ * @dfd: Base directory to pathwalk from *or* fd to stat.
+ * @filename: File to stat *or* NULL.
+ * @flags: AT_* flags to control pathwalk.
+ * @mask: Parts of statx struct actually required.
+ * @buffer: Result buffer.
+ *
+ * Note that if filename is NULL, then it does the equivalent of fstat() using
+ * dfd to indicate the file of interest.
+ */
+SYSCALL_DEFINE5(statx,
+ int, dfd, const char __user *, filename, unsigned, flags,
+ unsigned int, mask,
+ struct statx __user *, buffer)
+{
+ struct kstat stat;
+ int error;
+
+ if ((flags & AT_STATX_SYNC_TYPE) == AT_STATX_SYNC_TYPE)
+ return -EINVAL;
+ if (!access_ok(VERIFY_WRITE, buffer, sizeof(*buffer)))
+ return -EFAULT;
+
+ if (filename)
+ error = vfs_statx(dfd, filename, flags, &stat, mask);
+ else
+ error = vfs_statx_fd(dfd, &stat, mask, flags);
+ if (error)
+ return error;
+ return statx_set_result(&stat, buffer);
+}
+
/* Caller is here responsible for sufficient locking (ie. inode->i_lock) */
void __inode_add_bytes(struct inode *inode, loff_t bytes)
{
spin_unlock(&inode->i_lock);
mark_inode_dirty_sync(inode);
}
- return file->f_op->fsync(file, start, end, datasync);
+ return call_fsync(file, start, end, datasync);
}
EXPORT_SYMBOL(vfs_fsync_range);
return blocks;
}
-int sysv_getattr(struct vfsmount *mnt, struct dentry *dentry, struct kstat *stat)
+int sysv_getattr(const struct path *path, struct kstat *stat,
+ u32 request_mask, unsigned int flags)
{
- struct super_block *s = dentry->d_sb;
- generic_fillattr(d_inode(dentry), stat);
+ struct super_block *s = path->dentry->d_sb;
+ generic_fillattr(d_inode(path->dentry), stat);
stat->blocks = (s->s_blocksize / 512) * sysv_nblocks(s, stat->size);
stat->blksize = s->s_blocksize;
return 0;
extern int sysv_write_inode(struct inode *, struct writeback_control *wbc);
extern int sysv_sync_inode(struct inode *);
extern void sysv_set_inode(struct inode *, dev_t);
-extern int sysv_getattr(struct vfsmount *, struct dentry *, struct kstat *);
+extern int sysv_getattr(const struct path *, struct kstat *, u32, unsigned int);
extern int sysv_init_icache(void);
extern void sysv_destroy_icache(void);
clockid != CLOCK_BOOTTIME_ALARM))
return -EINVAL;
- if (!capable(CAP_WAKE_ALARM) &&
- (clockid == CLOCK_REALTIME_ALARM ||
- clockid == CLOCK_BOOTTIME_ALARM))
+ if ((clockid == CLOCK_REALTIME_ALARM ||
+ clockid == CLOCK_BOOTTIME_ALARM) &&
+ !capable(CAP_WAKE_ALARM))
return -EPERM;
ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);
return ret;
ctx = f.file->private_data;
- if (!capable(CAP_WAKE_ALARM) && isalarm(ctx)) {
+ if (isalarm(ctx) && !capable(CAP_WAKE_ALARM)) {
fdput(f);
return -EPERM;
}
return do_rename(old_dir, old_dentry, new_dir, new_dentry, flags);
}
-int ubifs_getattr(struct vfsmount *mnt, struct dentry *dentry,
- struct kstat *stat)
+int ubifs_getattr(const struct path *path, struct kstat *stat,
+ u32 request_mask, unsigned int flags)
{
loff_t size;
- struct inode *inode = d_inode(dentry);
+ struct inode *inode = d_inode(path->dentry);
struct ubifs_inode *ui = ubifs_inode(inode);
mutex_lock(&ui->ui_mutex);
/* dir.c */
struct inode *ubifs_new_inode(struct ubifs_info *c, struct inode *dir,
umode_t mode);
-int ubifs_getattr(struct vfsmount *mnt, struct dentry *dentry,
- struct kstat *stat);
+int ubifs_getattr(const struct path *path, struct kstat *stat,
+ u32 request_mask, unsigned int flags);
int ubifs_check_dir_empty(struct inode *dir);
/* xattr.c */
return err;
}
-static int udf_symlink_getattr(struct vfsmount *mnt, struct dentry *dentry,
- struct kstat *stat)
+static int udf_symlink_getattr(const struct path *path, struct kstat *stat,
+ u32 request_mask, unsigned int flags)
{
+ struct dentry *dentry = path->dentry;
struct inode *inode = d_backing_inode(dentry);
struct page *page;
#include <linux/list.h>
#include <linux/hashtable.h>
-#include <linux/sched.h>
+#include <linux/sched/signal.h>
+#include <linux/sched/mm.h>
#include <linux/mm.h>
#include <linux/poll.h>
#include <linux/slab.h>
* userfaultfd_ctx_get - Acquires a reference to the internal userfaultfd
* context.
* @ctx: [in] Pointer to the userfaultfd context.
- *
- * Returns: In case of success, returns not zero.
*/
static void userfaultfd_ctx_get(struct userfaultfd_ctx *ctx)
{
{
struct mm_struct *mm = ctx->mm;
pgd_t *pgd;
+ p4d_t *p4d;
pud_t *pud;
pmd_t *pmd, _pmd;
pte_t *pte;
pgd = pgd_offset(mm, address);
if (!pgd_present(*pgd))
goto out;
- pud = pud_offset(pgd, address);
+ p4d = p4d_offset(pgd, address);
+ if (!p4d_present(*p4d))
+ goto out;
+ pud = pud_offset(p4d, address);
if (!pud_present(*pud))
goto out;
pmd = pmd_offset(pud, address);
* in such case.
*/
down_read(&mm->mmap_sem);
- ret = 0;
+ ret = VM_FAULT_NOPAGE;
}
}
return ret;
}
-static int userfaultfd_event_wait_completion(struct userfaultfd_ctx *ctx,
- struct userfaultfd_wait_queue *ewq)
+static void userfaultfd_event_wait_completion(struct userfaultfd_ctx *ctx,
+ struct userfaultfd_wait_queue *ewq)
{
- int ret = 0;
+ if (WARN_ON_ONCE(current->flags & PF_EXITING))
+ goto out;
ewq->ctx = ctx;
init_waitqueue_entry(&ewq->wq, current);
break;
if (ACCESS_ONCE(ctx->released) ||
fatal_signal_pending(current)) {
- ret = -1;
__remove_wait_queue(&ctx->event_wqh, &ewq->wq);
+ if (ewq->msg.event == UFFD_EVENT_FORK) {
+ struct userfaultfd_ctx *new;
+
+ new = (struct userfaultfd_ctx *)
+ (unsigned long)
+ ewq->msg.arg.reserved.reserved1;
+
+ userfaultfd_ctx_put(new);
+ }
break;
}
* ctx may go away after this if the userfault pseudo fd is
* already released.
*/
-
+out:
userfaultfd_ctx_put(ctx);
- return ret;
}
static void userfaultfd_event_complete(struct userfaultfd_ctx *ctx,
return 0;
}
-static int dup_fctx(struct userfaultfd_fork_ctx *fctx)
+static void dup_fctx(struct userfaultfd_fork_ctx *fctx)
{
struct userfaultfd_ctx *ctx = fctx->orig;
struct userfaultfd_wait_queue ewq;
ewq.msg.event = UFFD_EVENT_FORK;
ewq.msg.arg.reserved.reserved1 = (unsigned long)fctx->new;
- return userfaultfd_event_wait_completion(ctx, &ewq);
+ userfaultfd_event_wait_completion(ctx, &ewq);
}
void dup_userfaultfd_complete(struct list_head *fcs)
{
- int ret = 0;
struct userfaultfd_fork_ctx *fctx, *n;
list_for_each_entry_safe(fctx, n, fcs, list) {
- if (!ret)
- ret = dup_fctx(fctx);
+ dup_fctx(fctx);
list_del(&fctx->list);
kfree(fctx);
}
userfaultfd_event_wait_completion(ctx, &ewq);
}
-void userfaultfd_remove(struct vm_area_struct *vma,
- struct vm_area_struct **prev,
+bool userfaultfd_remove(struct vm_area_struct *vma,
unsigned long start, unsigned long end)
{
struct mm_struct *mm = vma->vm_mm;
ctx = vma->vm_userfaultfd_ctx.ctx;
if (!ctx || !(ctx->features & UFFD_FEATURE_EVENT_REMOVE))
- return;
+ return true;
userfaultfd_ctx_get(ctx);
up_read(&mm->mmap_sem);
- *prev = NULL; /* We wait for ACK w/o the mmap semaphore */
-
msg_init(&ewq.msg);
ewq.msg.event = UFFD_EVENT_REMOVE;
userfaultfd_event_wait_completion(ctx, &ewq);
- down_read(&mm->mmap_sem);
+ return false;
}
static bool has_unmap_ctx(struct userfaultfd_ctx *ctx, struct list_head *unmaps,
}
}
-void userfaultfd_exit(struct mm_struct *mm)
-{
- struct vm_area_struct *vma = mm->mmap;
-
- /*
- * We can do the vma walk without locking because the caller
- * (exit_mm) knows it now has exclusive access
- */
- while (vma) {
- struct userfaultfd_ctx *ctx = vma->vm_userfaultfd_ctx.ctx;
-
- if (ctx && (ctx->features & UFFD_FEATURE_EVENT_EXIT)) {
- struct userfaultfd_wait_queue ewq;
-
- userfaultfd_ctx_get(ctx);
-
- msg_init(&ewq.msg);
- ewq.msg.event = UFFD_EVENT_EXIT;
-
- userfaultfd_event_wait_completion(ctx, &ewq);
-
- ctx->features &= ~UFFD_FEATURE_EVENT_EXIT;
- }
-
- vma = vma->vm_next;
- }
-}
-
static int userfaultfd_release(struct inode *inode, struct file *file)
{
struct userfaultfd_ctx *ctx = file->private_data;
* Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
*/
#include <linux/mm.h>
+#include <linux/sched/mm.h>
#include <linux/highmem.h>
#include <linux/slab.h>
#include <linux/swap.h>
#include "kmem.h"
#include "xfs_message.h"
-/*
- * Greedy allocation. May fail and may return vmalloced memory.
- */
-void *
-kmem_zalloc_greedy(size_t *size, size_t minsize, size_t maxsize)
-{
- void *ptr;
- size_t kmsize = maxsize;
-
- while (!(ptr = vzalloc(kmsize))) {
- if ((kmsize >>= 1) <= minsize)
- kmsize = minsize;
- }
- if (ptr)
- *size = kmsize;
- return ptr;
-}
-
void *
kmem_alloc(size_t size, xfs_km_flags_t flags)
{
}
-extern void *kmem_zalloc_greedy(size_t *, size_t, size_t);
-
static inline void *
kmem_zalloc(size_t size, xfs_km_flags_t flags)
{
args.type = XFS_ALLOCTYPE_START_BNO;
args.fsbno = XFS_INO_TO_FSB(mp, ip->i_ino);
} else if (dfops->dop_low) {
-try_another_ag:
args.type = XFS_ALLOCTYPE_START_BNO;
+try_another_ag:
args.fsbno = *firstblock;
} else {
args.type = XFS_ALLOCTYPE_NEAR_BNO;
if (xfs_sb_version_hasreflink(&cur->bc_mp->m_sb) &&
args.fsbno == NULLFSBLOCK &&
args.type == XFS_ALLOCTYPE_NEAR_BNO) {
- dfops->dop_low = true;
+ args.type = XFS_ALLOCTYPE_FIRST_AG;
goto try_another_ag;
}
+ if (WARN_ON_ONCE(args.fsbno == NULLFSBLOCK)) {
+ xfs_iroot_realloc(ip, -1, whichfork);
+ xfs_btree_del_cursor(cur, XFS_BTREE_ERROR);
+ return -ENOSPC;
+ }
/*
* Allocation can't fail, the space was reserved.
*/
- ASSERT(args.fsbno != NULLFSBLOCK);
ASSERT(*firstblock == NULLFSBLOCK ||
args.agno >= XFS_FSB_TO_AGNO(mp, *firstblock));
*firstblock = cur->bc_private.b.firstblock = args.fsbno;
return 0;
}
+/*
+ * Add a delayed allocation extent to an inode. Blocks are reserved from the
+ * global pool and the extent inserted into the inode in-core extent tree.
+ *
+ * On entry, got refers to the first extent beyond the offset of the extent to
+ * allocate or eof is specified if no such extent exists. On return, got refers
+ * to the extent record that was inserted to the inode fork.
+ *
+ * Note that the allocated extent may have been merged with contiguous extents
+ * during insertion into the inode fork. Thus, got does not reflect the current
+ * state of the inode fork on return. If necessary, the caller can use lastx to
+ * look up the updated record in the inode fork.
+ */
int
xfs_bmapi_reserve_delalloc(
struct xfs_inode *ip,
got->br_startblock = nullstartblock(indlen);
got->br_blockcount = alen;
got->br_state = XFS_EXT_NORM;
- xfs_bmap_add_extent_hole_delay(ip, whichfork, lastx, got);
- /*
- * Update our extent pointer, given that xfs_bmap_add_extent_hole_delay
- * might have merged it into one of the neighbouring ones.
- */
- xfs_bmbt_get_all(xfs_iext_get_ext(ifp, *lastx), got);
+ xfs_bmap_add_extent_hole_delay(ip, whichfork, lastx, got);
/*
* Tag the inode if blocks were preallocated. Note that COW fork
if (whichfork == XFS_COW_FORK && (prealloc || aoff < off || alen > len))
xfs_inode_set_cowblocks_tag(ip);
- ASSERT(got->br_startoff <= aoff);
- ASSERT(got->br_startoff + got->br_blockcount >= aoff + alen);
- ASSERT(isnullstartblock(got->br_startblock));
- ASSERT(got->br_state == XFS_EXT_NORM);
return 0;
out_unreserve_blocks:
if (args.fsbno == NULLFSBLOCK) {
args.fsbno = be64_to_cpu(start->l);
-try_another_ag:
args.type = XFS_ALLOCTYPE_START_BNO;
+try_another_ag:
/*
* Make sure there is sufficient room left in the AG to
* complete a full tree split for an extent insert. If
if (xfs_sb_version_hasreflink(&cur->bc_mp->m_sb) &&
args.fsbno == NULLFSBLOCK &&
args.type == XFS_ALLOCTYPE_NEAR_BNO) {
- cur->bc_private.b.dfops->dop_low = true;
args.fsbno = cur->bc_private.b.firstblock;
+ args.type = XFS_ALLOCTYPE_FIRST_AG;
goto try_another_ag;
}
goto error0;
cur->bc_private.b.dfops->dop_low = true;
}
- if (args.fsbno == NULLFSBLOCK) {
+ if (WARN_ON_ONCE(args.fsbno == NULLFSBLOCK)) {
XFS_BTREE_TRACE_CURSOR(cur, XBT_EXIT);
*stat = 0;
return 0;
struct xfs_ioend *ioend =
container_of(work, struct xfs_ioend, io_work);
struct xfs_inode *ip = XFS_I(ioend->io_inode);
+ xfs_off_t offset = ioend->io_offset;
+ size_t size = ioend->io_size;
int error = ioend->io_bio->bi_error;
/*
- * Set an error if the mount has shut down and proceed with end I/O
- * processing so it can perform whatever cleanups are necessary.
+ * Just clean up the in-memory strutures if the fs has been shut down.
*/
- if (XFS_FORCED_SHUTDOWN(ip->i_mount))
+ if (XFS_FORCED_SHUTDOWN(ip->i_mount)) {
error = -EIO;
+ goto done;
+ }
/*
- * For a CoW extent, we need to move the mapping from the CoW fork
- * to the data fork. If instead an error happened, just dump the
- * new blocks.
+ * Clean up any COW blocks on an I/O error.
*/
- if (ioend->io_type == XFS_IO_COW) {
- if (error)
- goto done;
- if (ioend->io_bio->bi_error) {
- error = xfs_reflink_cancel_cow_range(ip,
- ioend->io_offset, ioend->io_size);
- goto done;
+ if (unlikely(error)) {
+ switch (ioend->io_type) {
+ case XFS_IO_COW:
+ xfs_reflink_cancel_cow_range(ip, offset, size, true);
+ break;
}
- error = xfs_reflink_end_cow(ip, ioend->io_offset,
- ioend->io_size);
- if (error)
- goto done;
+
+ goto done;
}
/*
- * For unwritten extents we need to issue transactions to convert a
- * range to normal written extens after the data I/O has finished.
- * Detecting and handling completion IO errors is done individually
- * for each case as different cleanup operations need to be performed
- * on error.
+ * Success: commit the COW or unwritten blocks if needed.
*/
- if (ioend->io_type == XFS_IO_UNWRITTEN) {
- if (error)
- goto done;
- error = xfs_iomap_write_unwritten(ip, ioend->io_offset,
- ioend->io_size);
- } else if (ioend->io_append_trans) {
- error = xfs_setfilesize_ioend(ioend, error);
- } else {
- ASSERT(!xfs_ioend_is_append(ioend) ||
- ioend->io_type == XFS_IO_COW);
+ switch (ioend->io_type) {
+ case XFS_IO_COW:
+ error = xfs_reflink_end_cow(ip, offset, size);
+ break;
+ case XFS_IO_UNWRITTEN:
+ error = xfs_iomap_write_unwritten(ip, offset, size);
+ break;
+ default:
+ ASSERT(!xfs_ioend_is_append(ioend) || ioend->io_append_trans);
+ break;
}
done:
+ if (ioend->io_append_trans)
+ error = xfs_setfilesize_ioend(ioend, error);
xfs_destroy_ioend(ioend, error);
}
#include <linux/migrate.h>
#include <linux/backing-dev.h>
#include <linux/freezer.h>
+#include <linux/sched/mm.h>
#include "xfs_format.h"
#include "xfs_log_format.h"
xfs_ilock(ip, XFS_IOLOCK_EXCL);
xfs_ilock(ip, XFS_MMAPLOCK_EXCL);
- ret = xfs_reflink_cancel_cow_range(ip, 0, NULLFILEOFF);
+ ret = xfs_reflink_cancel_cow_range(ip, 0, NULLFILEOFF, false);
xfs_iunlock(ip, XFS_MMAPLOCK_EXCL);
xfs_iunlock(ip, XFS_IOLOCK_EXCL);
/* Remove all pending CoW reservations. */
error = xfs_reflink_cancel_cow_blocks(ip, &tp, first_unmap_block,
- last_block);
+ last_block, true);
if (error)
goto out;
#include "xfs_acl.h"
#include <linux/capability.h>
+#include <linux/cred.h>
#include <linux/dcache.h>
#include <linux/mount.h>
#include <linux/namei.h>
goto out_unlock;
}
+ /*
+ * Flag newly allocated delalloc blocks with IOMAP_F_NEW so we punch
+ * them out if the write happens to fail.
+ */
+ iomap->flags = IOMAP_F_NEW;
trace_xfs_iomap_alloc(ip, offset, count, 0, &got);
done:
if (isnullstartblock(got.br_startblock))
struct xfs_inode *ip,
loff_t offset,
loff_t length,
- ssize_t written)
+ ssize_t written,
+ struct iomap *iomap)
{
struct xfs_mount *mp = ip->i_mount;
xfs_fileoff_t start_fsb;
xfs_fileoff_t end_fsb;
int error = 0;
- /* behave as if the write failed if drop writes is enabled */
- if (xfs_mp_drop_writes(mp))
+ /*
+ * Behave as if the write failed if drop writes is enabled. Set the NEW
+ * flag to force delalloc cleanup.
+ */
+ if (xfs_mp_drop_writes(mp)) {
+ iomap->flags |= IOMAP_F_NEW;
written = 0;
+ }
/*
* start_fsb refers to the first unused block after a short write. If
end_fsb = XFS_B_TO_FSB(mp, offset + length);
/*
- * Trim back delalloc blocks if we didn't manage to write the whole
- * range reserved.
+ * Trim delalloc blocks if they were allocated by this write and we
+ * didn't manage to write the whole range.
*
* We don't need to care about racing delalloc as we hold i_mutex
* across the reserve/allocate/unreserve calls. If there are delalloc
* blocks in the range, they are ours.
*/
- if (start_fsb < end_fsb) {
+ if ((iomap->flags & IOMAP_F_NEW) && start_fsb < end_fsb) {
truncate_pagecache_range(VFS_I(ip), XFS_FSB_TO_B(mp, start_fsb),
XFS_FSB_TO_B(mp, end_fsb) - 1);
{
if ((flags & IOMAP_WRITE) && iomap->type == IOMAP_DELALLOC)
return xfs_file_iomap_end_delalloc(XFS_I(inode), offset,
- length, written);
+ length, written, iomap);
return 0;
}
STATIC int
xfs_vn_getattr(
- struct vfsmount *mnt,
- struct dentry *dentry,
- struct kstat *stat)
+ const struct path *path,
+ struct kstat *stat,
+ u32 request_mask,
+ unsigned int query_flags)
{
- struct inode *inode = d_inode(dentry);
+ struct inode *inode = d_inode(path->dentry);
struct xfs_inode *ip = XFS_I(inode);
struct xfs_mount *mp = ip->i_mount;
xfs_agino_t agino; /* inode # in allocation group */
xfs_agnumber_t agno; /* allocation group number */
xfs_btree_cur_t *cur; /* btree cursor for ialloc btree */
- size_t irbsize; /* size of irec buffer in bytes */
xfs_inobt_rec_incore_t *irbuf; /* start of irec buffer */
int nirbuf; /* size of irbuf */
int ubcount; /* size of user's buffer */
*ubcountp = 0;
*done = 0;
- irbuf = kmem_zalloc_greedy(&irbsize, PAGE_SIZE, PAGE_SIZE * 4);
+ irbuf = kmem_zalloc_large(PAGE_SIZE * 4, KM_SLEEP);
if (!irbuf)
return -ENOMEM;
-
- nirbuf = irbsize / sizeof(*irbuf);
+ nirbuf = (PAGE_SIZE * 4) / sizeof(*irbuf);
/*
* Loop over the allocation groups, starting from the last
#include <linux/file.h>
#include <linux/swap.h>
#include <linux/errno.h>
-#include <linux/sched.h>
+#include <linux/sched/signal.h>
#include <linux/bitops.h>
#include <linux/major.h>
#include <linux/pagemap.h>
xfs_set_inoalignment(xfs_mount_t *mp)
{
if (xfs_sb_version_hasalign(&mp->m_sb) &&
- mp->m_sb.sb_inoalignmt >=
- XFS_B_TO_FSBT(mp, mp->m_inode_cluster_size))
+ mp->m_sb.sb_inoalignmt >= xfs_icluster_size_fsb(mp))
mp->m_inoalign_mask = mp->m_sb.sb_inoalignmt - 1;
else
mp->m_inoalign_mask = 0;
}
/*
- * Cancel all pending CoW reservations for some block range of an inode.
+ * Cancel CoW reservations for some block range of an inode.
+ *
+ * If cancel_real is true this function cancels all COW fork extents for the
+ * inode; if cancel_real is false, real extents are not cleared.
*/
int
xfs_reflink_cancel_cow_blocks(
struct xfs_inode *ip,
struct xfs_trans **tpp,
xfs_fileoff_t offset_fsb,
- xfs_fileoff_t end_fsb)
+ xfs_fileoff_t end_fsb,
+ bool cancel_real)
{
struct xfs_ifork *ifp = XFS_IFORK_PTR(ip, XFS_COW_FORK);
struct xfs_bmbt_irec got, del;
&idx, &got, &del);
if (error)
break;
- } else {
+ } else if (del.br_state == XFS_EXT_UNWRITTEN || cancel_real) {
xfs_trans_ijoin(*tpp, ip, 0);
xfs_defer_init(&dfops, &firstfsb);
}
/*
- * Cancel all pending CoW reservations for some byte range of an inode.
+ * Cancel CoW reservations for some byte range of an inode.
+ *
+ * If cancel_real is true this function cancels all COW fork extents for the
+ * inode; if cancel_real is false, real extents are not cleared.
*/
int
xfs_reflink_cancel_cow_range(
struct xfs_inode *ip,
xfs_off_t offset,
- xfs_off_t count)
+ xfs_off_t count,
+ bool cancel_real)
{
struct xfs_trans *tp;
xfs_fileoff_t offset_fsb;
xfs_trans_ijoin(tp, ip, 0);
/* Scrape out the old CoW reservations */
- error = xfs_reflink_cancel_cow_blocks(ip, &tp, offset_fsb, end_fsb);
+ error = xfs_reflink_cancel_cow_blocks(ip, &tp, offset_fsb, end_fsb,
+ cancel_real);
if (error)
goto out_cancel;
* We didn't find any shared blocks so turn off the reflink flag.
* First, get rid of any leftover CoW mappings.
*/
- error = xfs_reflink_cancel_cow_blocks(ip, tpp, 0, NULLFILEOFF);
+ error = xfs_reflink_cancel_cow_blocks(ip, tpp, 0, NULLFILEOFF, true);
if (error)
return error;
extern int xfs_reflink_cancel_cow_blocks(struct xfs_inode *ip,
struct xfs_trans **tpp, xfs_fileoff_t offset_fsb,
- xfs_fileoff_t end_fsb);
+ xfs_fileoff_t end_fsb, bool cancel_real);
extern int xfs_reflink_cancel_cow_range(struct xfs_inode *ip, xfs_off_t offset,
- xfs_off_t count);
+ xfs_off_t count, bool cancel_real);
extern int xfs_reflink_end_cow(struct xfs_inode *ip, xfs_off_t offset,
xfs_off_t count);
extern int xfs_reflink_recover_cow(struct xfs_mount *mp);
XFS_STATS_INC(ip->i_mount, vn_remove);
if (xfs_is_reflink_inode(ip)) {
- error = xfs_reflink_cancel_cow_range(ip, 0, NULLFILEOFF);
+ error = xfs_reflink_cancel_cow_range(ip, 0, NULLFILEOFF, true);
if (error && !XFS_FORCED_SHUTDOWN(ip->i_mount))
xfs_warn(ip->i_mount,
"Error %d while evicting CoW blocks for inode %llu.",
((unlikely(pgd_none(*(pud))) && __pmd_alloc(mm, pud, address))? \
NULL: pmd_offset(pud, address))
-#define pud_alloc(mm, pgd, address) (pgd)
#define pud_offset(pgd, start) (pgd)
#define pud_none(pud) 0
#define pud_bad(pud) 0
#undef pud_addr_end
#define pud_addr_end(addr, end) (end)
+#include <asm-generic/5level-fixup.h>
+
#endif
--- /dev/null
+#ifndef _5LEVEL_FIXUP_H
+#define _5LEVEL_FIXUP_H
+
+#define __ARCH_HAS_5LEVEL_HACK
+#define __PAGETABLE_P4D_FOLDED
+
+#define P4D_SHIFT PGDIR_SHIFT
+#define P4D_SIZE PGDIR_SIZE
+#define P4D_MASK PGDIR_MASK
+#define PTRS_PER_P4D 1
+
+#define p4d_t pgd_t
+
+#define pud_alloc(mm, p4d, address) \
+ ((unlikely(pgd_none(*(p4d))) && __pud_alloc(mm, p4d, address)) ? \
+ NULL : pud_offset(p4d, address))
+
+#define p4d_alloc(mm, pgd, address) (pgd)
+#define p4d_offset(pgd, start) (pgd)
+#define p4d_none(p4d) 0
+#define p4d_bad(p4d) 0
+#define p4d_present(p4d) 1
+#define p4d_ERROR(p4d) do { } while (0)
+#define p4d_clear(p4d) pgd_clear(p4d)
+#define p4d_val(p4d) pgd_val(p4d)
+#define p4d_populate(mm, p4d, pud) pgd_populate(mm, p4d, pud)
+#define p4d_page(p4d) pgd_page(p4d)
+#define p4d_page_vaddr(p4d) pgd_page_vaddr(p4d)
+
+#define __p4d(x) __pgd(x)
+#define set_p4d(p4dp, p4d) set_pgd(p4dp, p4d)
+
+#undef p4d_free_tlb
+#define p4d_free_tlb(tlb, x, addr) do { } while (0)
+#define p4d_free(mm, x) do { } while (0)
+#define __p4d_free_tlb(tlb, x, addr) do { } while (0)
+
+#undef p4d_addr_end
+#define p4d_addr_end(addr, end) (end)
+
+#endif
--- /dev/null
+#ifndef _PGTABLE_NOP4D_HACK_H
+#define _PGTABLE_NOP4D_HACK_H
+
+#ifndef __ASSEMBLY__
+#include <asm-generic/5level-fixup.h>
+
+#define __PAGETABLE_PUD_FOLDED
+
+/*
+ * Having the pud type consist of a pgd gets the size right, and allows
+ * us to conceptually access the pgd entry that this pud is folded into
+ * without casting.
+ */
+typedef struct { pgd_t pgd; } pud_t;
+
+#define PUD_SHIFT PGDIR_SHIFT
+#define PTRS_PER_PUD 1
+#define PUD_SIZE (1UL << PUD_SHIFT)
+#define PUD_MASK (~(PUD_SIZE-1))
+
+/*
+ * The "pgd_xxx()" functions here are trivial for a folded two-level
+ * setup: the pud is never bad, and a pud always exists (as it's folded
+ * into the pgd entry)
+ */
+static inline int pgd_none(pgd_t pgd) { return 0; }
+static inline int pgd_bad(pgd_t pgd) { return 0; }
+static inline int pgd_present(pgd_t pgd) { return 1; }
+static inline void pgd_clear(pgd_t *pgd) { }
+#define pud_ERROR(pud) (pgd_ERROR((pud).pgd))
+
+#define pgd_populate(mm, pgd, pud) do { } while (0)
+/*
+ * (puds are folded into pgds so this doesn't get actually called,
+ * but the define is needed for a generic inline function.)
+ */
+#define set_pgd(pgdptr, pgdval) set_pud((pud_t *)(pgdptr), (pud_t) { pgdval })
+
+static inline pud_t *pud_offset(pgd_t *pgd, unsigned long address)
+{
+ return (pud_t *)pgd;
+}
+
+#define pud_val(x) (pgd_val((x).pgd))
+#define __pud(x) ((pud_t) { __pgd(x) })
+
+#define pgd_page(pgd) (pud_page((pud_t){ pgd }))
+#define pgd_page_vaddr(pgd) (pud_page_vaddr((pud_t){ pgd }))
+
+/*
+ * allocating and freeing a pud is trivial: the 1-entry pud is
+ * inside the pgd, so has no extra memory associated with it.
+ */
+#define pud_alloc_one(mm, address) NULL
+#define pud_free(mm, x) do { } while (0)
+#define __pud_free_tlb(tlb, x, a) do { } while (0)
+
+#undef pud_addr_end
+#define pud_addr_end(addr, end) (end)
+
+#endif /* __ASSEMBLY__ */
+#endif /* _PGTABLE_NOP4D_HACK_H */
--- /dev/null
+#ifndef _PGTABLE_NOP4D_H
+#define _PGTABLE_NOP4D_H
+
+#ifndef __ASSEMBLY__
+
+#define __PAGETABLE_P4D_FOLDED
+
+typedef struct { pgd_t pgd; } p4d_t;
+
+#define P4D_SHIFT PGDIR_SHIFT
+#define PTRS_PER_P4D 1
+#define P4D_SIZE (1UL << P4D_SHIFT)
+#define P4D_MASK (~(P4D_SIZE-1))
+
+/*
+ * The "pgd_xxx()" functions here are trivial for a folded two-level
+ * setup: the p4d is never bad, and a p4d always exists (as it's folded
+ * into the pgd entry)
+ */
+static inline int pgd_none(pgd_t pgd) { return 0; }
+static inline int pgd_bad(pgd_t pgd) { return 0; }
+static inline int pgd_present(pgd_t pgd) { return 1; }
+static inline void pgd_clear(pgd_t *pgd) { }
+#define p4d_ERROR(p4d) (pgd_ERROR((p4d).pgd))
+
+#define pgd_populate(mm, pgd, p4d) do { } while (0)
+/*
+ * (p4ds are folded into pgds so this doesn't get actually called,
+ * but the define is needed for a generic inline function.)
+ */
+#define set_pgd(pgdptr, pgdval) set_p4d((p4d_t *)(pgdptr), (p4d_t) { pgdval })
+
+static inline p4d_t *p4d_offset(pgd_t *pgd, unsigned long address)
+{
+ return (p4d_t *)pgd;
+}
+
+#define p4d_val(x) (pgd_val((x).pgd))
+#define __p4d(x) ((p4d_t) { __pgd(x) })
+
+#define pgd_page(pgd) (p4d_page((p4d_t){ pgd }))
+#define pgd_page_vaddr(pgd) (p4d_page_vaddr((p4d_t){ pgd }))
+
+/*
+ * allocating and freeing a p4d is trivial: the 1-entry p4d is
+ * inside the pgd, so has no extra memory associated with it.
+ */
+#define p4d_alloc_one(mm, address) NULL
+#define p4d_free(mm, x) do { } while (0)
+#define __p4d_free_tlb(tlb, x, a) do { } while (0)
+
+#undef p4d_addr_end
+#define p4d_addr_end(addr, end) (end)
+
+#endif /* __ASSEMBLY__ */
+#endif /* _PGTABLE_NOP4D_H */
#ifndef __ASSEMBLY__
+#ifdef __ARCH_USE_5LEVEL_HACK
+#include <asm-generic/pgtable-nop4d-hack.h>
+#else
+#include <asm-generic/pgtable-nop4d.h>
+
#define __PAGETABLE_PUD_FOLDED
/*
- * Having the pud type consist of a pgd gets the size right, and allows
- * us to conceptually access the pgd entry that this pud is folded into
+ * Having the pud type consist of a p4d gets the size right, and allows
+ * us to conceptually access the p4d entry that this pud is folded into
* without casting.
*/
-typedef struct { pgd_t pgd; } pud_t;
+typedef struct { p4d_t p4d; } pud_t;
-#define PUD_SHIFT PGDIR_SHIFT
+#define PUD_SHIFT P4D_SHIFT
#define PTRS_PER_PUD 1
#define PUD_SIZE (1UL << PUD_SHIFT)
#define PUD_MASK (~(PUD_SIZE-1))
/*
- * The "pgd_xxx()" functions here are trivial for a folded two-level
+ * The "p4d_xxx()" functions here are trivial for a folded two-level
* setup: the pud is never bad, and a pud always exists (as it's folded
- * into the pgd entry)
+ * into the p4d entry)
*/
-static inline int pgd_none(pgd_t pgd) { return 0; }
-static inline int pgd_bad(pgd_t pgd) { return 0; }
-static inline int pgd_present(pgd_t pgd) { return 1; }
-static inline void pgd_clear(pgd_t *pgd) { }
-#define pud_ERROR(pud) (pgd_ERROR((pud).pgd))
+static inline int p4d_none(p4d_t p4d) { return 0; }
+static inline int p4d_bad(p4d_t p4d) { return 0; }
+static inline int p4d_present(p4d_t p4d) { return 1; }
+static inline void p4d_clear(p4d_t *p4d) { }
+#define pud_ERROR(pud) (p4d_ERROR((pud).p4d))
-#define pgd_populate(mm, pgd, pud) do { } while (0)
+#define p4d_populate(mm, p4d, pud) do { } while (0)
/*
- * (puds are folded into pgds so this doesn't get actually called,
+ * (puds are folded into p4ds so this doesn't get actually called,
* but the define is needed for a generic inline function.)
*/
-#define set_pgd(pgdptr, pgdval) set_pud((pud_t *)(pgdptr), (pud_t) { pgdval })
+#define set_p4d(p4dptr, p4dval) set_pud((pud_t *)(p4dptr), (pud_t) { p4dval })
-static inline pud_t * pud_offset(pgd_t * pgd, unsigned long address)
+static inline pud_t *pud_offset(p4d_t *p4d, unsigned long address)
{
- return (pud_t *)pgd;
+ return (pud_t *)p4d;
}
-#define pud_val(x) (pgd_val((x).pgd))
-#define __pud(x) ((pud_t) { __pgd(x) } )
+#define pud_val(x) (p4d_val((x).p4d))
+#define __pud(x) ((pud_t) { __p4d(x) })
-#define pgd_page(pgd) (pud_page((pud_t){ pgd }))
-#define pgd_page_vaddr(pgd) (pud_page_vaddr((pud_t){ pgd }))
+#define p4d_page(p4d) (pud_page((pud_t){ p4d }))
+#define p4d_page_vaddr(p4d) (pud_page_vaddr((pud_t){ p4d }))
/*
* allocating and freeing a pud is trivial: the 1-entry pud is
- * inside the pgd, so has no extra memory associated with it.
+ * inside the p4d, so has no extra memory associated with it.
*/
#define pud_alloc_one(mm, address) NULL
#define pud_free(mm, x) do { } while (0)
#define pud_addr_end(addr, end) (end)
#endif /* __ASSEMBLY__ */
+#endif /* !__ARCH_USE_5LEVEL_HACK */
#endif /* _PGTABLE_NOPUD_H */
#include <linux/bug.h>
#include <linux/errno.h>
-#if 4 - defined(__PAGETABLE_PUD_FOLDED) - defined(__PAGETABLE_PMD_FOLDED) != \
- CONFIG_PGTABLE_LEVELS
-#error CONFIG_PGTABLE_LEVELS is not consistent with __PAGETABLE_{PUD,PMD}_FOLDED
+#if 5 - defined(__PAGETABLE_P4D_FOLDED) - defined(__PAGETABLE_PUD_FOLDED) - \
+ defined(__PAGETABLE_PMD_FOLDED) != CONFIG_PGTABLE_LEVELS
+#error CONFIG_PGTABLE_LEVELS is not consistent with __PAGETABLE_{P4D,PUD,PMD}_FOLDED
#endif
/*
(__boundary - 1 < (end) - 1)? __boundary: (end); \
})
+#ifndef p4d_addr_end
+#define p4d_addr_end(addr, end) \
+({ unsigned long __boundary = ((addr) + P4D_SIZE) & P4D_MASK; \
+ (__boundary - 1 < (end) - 1)? __boundary: (end); \
+})
+#endif
+
#ifndef pud_addr_end
#define pud_addr_end(addr, end) \
({ unsigned long __boundary = ((addr) + PUD_SIZE) & PUD_MASK; \
* Do the tests inline, but report and clear the bad entry in mm/memory.c.
*/
void pgd_clear_bad(pgd_t *);
+void p4d_clear_bad(p4d_t *);
void pud_clear_bad(pud_t *);
void pmd_clear_bad(pmd_t *);
return 0;
}
+static inline int p4d_none_or_clear_bad(p4d_t *p4d)
+{
+ if (p4d_none(*p4d))
+ return 1;
+ if (unlikely(p4d_bad(*p4d))) {
+ p4d_clear_bad(p4d);
+ return 1;
+ }
+ return 0;
+}
+
static inline int pud_none_or_clear_bad(pud_t *pud)
{
if (pud_none(*pud))
#endif /* CONFIG_MMU */
#ifdef CONFIG_HAVE_ARCH_HUGE_VMAP
+
+#ifndef __PAGETABLE_P4D_FOLDED
+int p4d_set_huge(p4d_t *p4d, phys_addr_t addr, pgprot_t prot);
+int p4d_clear_huge(p4d_t *p4d);
+#else
+static inline int p4d_set_huge(p4d_t *p4d, phys_addr_t addr, pgprot_t prot)
+{
+ return 0;
+}
+static inline int p4d_clear_huge(p4d_t *p4d)
+{
+ return 0;
+}
+#endif /* !__PAGETABLE_P4D_FOLDED */
+
int pud_set_huge(pud_t *pud, phys_addr_t addr, pgprot_t prot);
int pmd_set_huge(pmd_t *pmd, phys_addr_t addr, pgprot_t prot);
int pud_clear_huge(pud_t *pud);
int pmd_clear_huge(pmd_t *pmd);
#else /* !CONFIG_HAVE_ARCH_HUGE_VMAP */
+static inline int p4d_set_huge(p4d_t *p4d, phys_addr_t addr, pgprot_t prot)
+{
+ return 0;
+}
static inline int pud_set_huge(pud_t *pud, phys_addr_t addr, pgprot_t prot)
{
return 0;
{
return 0;
}
+static inline int p4d_clear_huge(p4d_t *p4d)
+{
+ return 0;
+}
static inline int pud_clear_huge(pud_t *pud)
{
return 0;
__pte_free_tlb(tlb, ptep, address); \
} while (0)
+#define pmd_free_tlb(tlb, pmdp, address) \
+ do { \
+ __tlb_adjust_range(tlb, address, PAGE_SIZE); \
+ __pmd_free_tlb(tlb, pmdp, address); \
+ } while (0)
+
#ifndef __ARCH_HAS_4LEVEL_HACK
#define pud_free_tlb(tlb, pudp, address) \
do { \
} while (0)
#endif
-#define pmd_free_tlb(tlb, pmdp, address) \
+#ifndef __ARCH_HAS_5LEVEL_HACK
+#define p4d_free_tlb(tlb, pudp, address) \
do { \
- __tlb_adjust_range(tlb, address, PAGE_SIZE); \
- __pmd_free_tlb(tlb, pmdp, address); \
+ __tlb_adjust_range(tlb, address, PAGE_SIZE); \
+ __p4d_free_tlb(tlb, pudp, address); \
} while (0)
+#endif
#define tlb_migrate_finish(mm) do {} while (0)
return crypto_attr_alg(tb[1], type, mask);
}
+static inline int crypto_requires_off(u32 type, u32 mask, u32 off)
+{
+ return (type ^ off) & mask & off;
+}
+
/*
* Returns CRYPTO_ALG_ASYNC if type/mask requires the use of sync algorithms.
* Otherwise returns zero.
*/
static inline int crypto_requires_sync(u32 type, u32 mask)
{
- return (type ^ CRYPTO_ALG_ASYNC) & mask & CRYPTO_ALG_ASYNC;
+ return crypto_requires_off(type, mask, CRYPTO_ALG_ASYNC);
}
noinline unsigned long __crypto_memneq(const void *a, const void *b, size_t size);
#include <linux/bug.h>
#include <linux/rbtree.h>
#include <linux/kernel.h>
+#include <linux/mm_types.h>
#include <linux/list.h>
#include <linux/spinlock.h>
#ifdef CONFIG_DRM_DEBUG_MM
*/
#include <linux/interrupt.h> /* For task queue support */
+#include <linux/sched/signal.h>
#include <linux/delay.h>
#ifndef readq
#define CLKID_MPLL2 15
#define CLKID_SPI 34
#define CLKID_I2C 22
+#define CLKID_SAR_ADC 23
#define CLKID_ETH 36
#define CLKID_USB0 50
#define CLKID_USB1 51
#define CLKID_USB 55
+#define CLKID_HDMI_PCLK 63
#define CLKID_USB1_DDR_BRIDGE 64
#define CLKID_USB0_DDR_BRIDGE 65
+#define CLKID_SANA 69
+#define CLKID_GCLK_VENCI_INT0 77
#define CLKID_AO_I2C 93
#define CLKID_SD_EMMC_A 94
#define CLKID_SD_EMMC_B 95
#define CLKID_SD_EMMC_C 96
+#define CLKID_SAR_ADC_CLK 97
+#define CLKID_SAR_ADC_SEL 98
#endif /* __GXBB_CLKC_H */
#define EXYNOS_PIN_FUNC_6 6
#define EXYNOS_PIN_FUNC_F 0xf
+/* Drive strengths for Exynos7 FSYS1 block */
+#define EXYNOS7_FSYS1_PIN_DRV_LV1 0
+#define EXYNOS7_FSYS1_PIN_DRV_LV2 4
+#define EXYNOS7_FSYS1_PIN_DRV_LV3 2
+#define EXYNOS7_FSYS1_PIN_DRV_LV4 6
+#define EXYNOS7_FSYS1_PIN_DRV_LV5 1
+#define EXYNOS7_FSYS1_PIN_DRV_LV6 5
+
#endif /* __DT_BINDINGS_PINCTRL_SAMSUNG_H__ */
#define CS42L42_HPOUT_LOAD_1NF 0
#define CS42L42_HPOUT_LOAD_10NF 1
-/* HPOUT Clamp to GND Overide */
+/* HPOUT Clamp to GND Override */
#define CS42L42_HPOUT_CLAMP_EN 0
#define CS42L42_HPOUT_CLAMP_DIS 1
extern long user_read(const struct key *key,
char __user *buffer, size_t buflen);
-static inline const struct user_key_payload *user_key_payload(const struct key *key)
+static inline const struct user_key_payload *user_key_payload_rcu(const struct key *key)
{
- return (struct user_key_payload *)rcu_dereference_key(key);
+ return (struct user_key_payload *)dereference_key_rcu(key);
+}
+
+static inline struct user_key_payload *user_key_payload_locked(const struct key *key)
+{
+ return (struct user_key_payload *)dereference_key_locked((struct key *)key);
}
#endif /* CONFIG_KEYS */
#ifndef _LINUX_AVERAGE_H
#define _LINUX_AVERAGE_H
-/* Exponentially weighted moving average (EWMA) */
+/*
+ * Exponentially weighted moving average (EWMA)
+ *
+ * This implements a fixed-precision EWMA algorithm, with both the
+ * precision and fall-off coefficient determined at compile-time
+ * and built into the generated helper funtions.
+ *
+ * The first argument to the macro is the name that will be used
+ * for the struct and helper functions.
+ *
+ * The second argument, the precision, expresses how many bits are
+ * used for the fractional part of the fixed-precision values.
+ *
+ * The third argument, the weight reciprocal, determines how the
+ * new values will be weighed vs. the old state, new values will
+ * get weight 1/weight_rcp and old values 1-1/weight_rcp. Note
+ * that this parameter must be a power of two for efficiency.
+ */
-#define DECLARE_EWMA(name, _factor, _weight) \
+#define DECLARE_EWMA(name, _precision, _weight_rcp) \
struct ewma_##name { \
unsigned long internal; \
}; \
static inline void ewma_##name##_init(struct ewma_##name *e) \
{ \
- BUILD_BUG_ON(!__builtin_constant_p(_factor)); \
- BUILD_BUG_ON(!__builtin_constant_p(_weight)); \
- BUILD_BUG_ON_NOT_POWER_OF_2(_factor); \
- BUILD_BUG_ON_NOT_POWER_OF_2(_weight); \
+ BUILD_BUG_ON(!__builtin_constant_p(_precision)); \
+ BUILD_BUG_ON(!__builtin_constant_p(_weight_rcp)); \
+ /* \
+ * Even if you want to feed it just 0/1 you should have \
+ * some bits for the non-fractional part... \
+ */ \
+ BUILD_BUG_ON((_precision) > 30); \
+ BUILD_BUG_ON_NOT_POWER_OF_2(_weight_rcp); \
e->internal = 0; \
} \
static inline unsigned long \
ewma_##name##_read(struct ewma_##name *e) \
{ \
- BUILD_BUG_ON(!__builtin_constant_p(_factor)); \
- BUILD_BUG_ON(!__builtin_constant_p(_weight)); \
- BUILD_BUG_ON_NOT_POWER_OF_2(_factor); \
- BUILD_BUG_ON_NOT_POWER_OF_2(_weight); \
- return e->internal >> ilog2(_factor); \
+ BUILD_BUG_ON(!__builtin_constant_p(_precision)); \
+ BUILD_BUG_ON(!__builtin_constant_p(_weight_rcp)); \
+ BUILD_BUG_ON((_precision) > 30); \
+ BUILD_BUG_ON_NOT_POWER_OF_2(_weight_rcp); \
+ return e->internal >> (_precision); \
} \
static inline void ewma_##name##_add(struct ewma_##name *e, \
unsigned long val) \
{ \
unsigned long internal = ACCESS_ONCE(e->internal); \
- unsigned long weight = ilog2(_weight); \
- unsigned long factor = ilog2(_factor); \
+ unsigned long weight_rcp = ilog2(_weight_rcp); \
+ unsigned long precision = _precision; \
\
- BUILD_BUG_ON(!__builtin_constant_p(_factor)); \
- BUILD_BUG_ON(!__builtin_constant_p(_weight)); \
- BUILD_BUG_ON_NOT_POWER_OF_2(_factor); \
- BUILD_BUG_ON_NOT_POWER_OF_2(_weight); \
+ BUILD_BUG_ON(!__builtin_constant_p(_precision)); \
+ BUILD_BUG_ON(!__builtin_constant_p(_weight_rcp)); \
+ BUILD_BUG_ON((_precision) > 30); \
+ BUILD_BUG_ON_NOT_POWER_OF_2(_weight_rcp); \
\
ACCESS_ONCE(e->internal) = internal ? \
- (((internal << weight) - internal) + \
- (val << factor)) >> weight : \
- (val << factor); \
+ (((internal << weight_rcp) - internal) + \
+ (val << precision)) >> weight_rcp : \
+ (val << precision); \
}
#endif /* _LINUX_AVERAGE_H */
#include <asm/exec.h>
#include <uapi/linux/binfmts.h>
+struct filename;
+
#define CORENAME_MAX_SIZE 128
/*
extern void set_binfmt(struct linux_binfmt *new);
extern ssize_t read_code(struct file *, unsigned long, loff_t, size_t);
+extern int do_execve(struct filename *,
+ const char __user * const __user *,
+ const char __user * const __user *);
+extern int do_execveat(int, struct filename *,
+ const char __user * const __user *,
+ const char __user * const __user *,
+ int);
+
#endif /* _LINUX_BINFMTS_H */
--- /dev/null
+#ifndef _LINUX_BLK_MQ_VIRTIO_H
+#define _LINUX_BLK_MQ_VIRTIO_H
+
+struct blk_mq_tag_set;
+struct virtio_device;
+
+int blk_mq_virtio_map_queues(struct blk_mq_tag_set *set,
+ struct virtio_device *vdev, int first_vec);
+
+#endif /* _LINUX_BLK_MQ_VIRTIO_H */
void blk_mq_freeze_queue(struct request_queue *q);
void blk_mq_unfreeze_queue(struct request_queue *q);
void blk_mq_freeze_queue_start(struct request_queue *q);
+void blk_mq_freeze_queue_wait(struct request_queue *q);
+int blk_mq_freeze_queue_wait_timeout(struct request_queue *q,
+ unsigned long timeout);
int blk_mq_reinit_tagset(struct blk_mq_tag_set *set);
int blk_mq_map_queues(struct blk_mq_tag_set *set);
#define _LINUX_BLKDEV_H
#include <linux/sched.h>
+#include <linux/sched/clock.h>
#ifdef CONFIG_BLOCK
struct delayed_work delay_work;
struct backing_dev_info *backing_dev_info;
- struct disk_devt *disk_devt;
/*
* The queue owner gets to use this for whatever they like.
unsigned long mount_timeout; /* jiffies */
unsigned long osd_idle_ttl; /* jiffies */
unsigned long osd_keepalive_timeout; /* jiffies */
+ unsigned long osd_request_timeout; /* jiffies */
/*
* any type that can't be simply compared or doesn't need need
#define CEPH_MOUNT_TIMEOUT_DEFAULT msecs_to_jiffies(60 * 1000)
#define CEPH_OSD_KEEPALIVE_DEFAULT msecs_to_jiffies(5 * 1000)
#define CEPH_OSD_IDLE_TTL_DEFAULT msecs_to_jiffies(60 * 1000)
+#define CEPH_OSD_REQUEST_TIMEOUT_DEFAULT 0 /* no timeout */
#define CEPH_MONC_HUNT_INTERVAL msecs_to_jiffies(3 * 1000)
#define CEPH_MONC_PING_INTERVAL msecs_to_jiffies(10 * 1000)
/* internal */
unsigned long r_stamp; /* jiffies, send or check time */
+ unsigned long r_start_stamp; /* jiffies */
int r_attempts;
struct ceph_eversion r_replay_version; /* aka reassert_version */
u32 r_last_force_resend;
* cgroup_threadgroup_change_begin - threadgroup exclusion for cgroups
* @tsk: target task
*
- * Called from threadgroup_change_begin() and allows cgroup operations to
- * synchronize against threadgroup changes using a percpu_rw_semaphore.
+ * Allows cgroup operations to synchronize against threadgroup changes
+ * using a percpu_rw_semaphore.
*/
static inline void cgroup_threadgroup_change_begin(struct task_struct *tsk)
{
* cgroup_threadgroup_change_end - threadgroup exclusion for cgroups
* @tsk: target task
*
- * Called from threadgroup_change_end(). Counterpart of
- * cgroup_threadcgroup_change_begin().
+ * Counterpart of cgroup_threadcgroup_change_begin().
*/
static inline void cgroup_threadgroup_change_end(struct task_struct *tsk)
{
#define CGROUP_SUBSYS_COUNT 0
-static inline void cgroup_threadgroup_change_begin(struct task_struct *tsk) {}
+static inline void cgroup_threadgroup_change_begin(struct task_struct *tsk)
+{
+ might_sleep();
+}
+
static inline void cgroup_threadgroup_change_end(struct task_struct *tsk) {}
#endif /* CONFIG_CGROUPS */
extern void boot_cpu_init(void);
extern void boot_cpu_state_init(void);
+extern void cpu_init(void);
+extern void trap_init(void);
extern int register_cpu(struct cpu *cpu, int num);
extern struct device *get_cpu_device(unsigned cpu);
CPUHP_ARM_OMAP_WAKE_DEAD,
CPUHP_IRQ_POLL_DEAD,
CPUHP_BLOCK_SOFTIRQ_DEAD,
- CPUHP_VIRT_SCSI_DEAD,
CPUHP_ACPI_CPUDRV_DEAD,
CPUHP_S390_PFAULT_DEAD,
CPUHP_BLK_MQ_DEAD,
*/
#include <linux/sched.h>
+#include <linux/sched/topology.h>
+#include <linux/sched/task.h>
#include <linux/cpumask.h>
#include <linux/nodemask.h>
#include <linux/mm.h>
+++ /dev/null
-#ifndef __LINUX_CPUTIME_H
-#define __LINUX_CPUTIME_H
-
-#ifdef CONFIG_VIRT_CPU_ACCOUNTING_NATIVE
-#include <asm/cputime.h>
-
-#ifndef cputime_to_nsecs
-# define cputime_to_nsecs(__ct) \
- (cputime_to_usecs(__ct) * NSEC_PER_USEC)
-#endif
-
-#endif /* CONFIG_VIRT_CPU_ACCOUNTING_NATIVE */
-#endif /* __LINUX_CPUTIME_H */
#include <linux/selinux.h>
#include <linux/atomic.h>
#include <linux/uidgid.h>
+#include <linux/sched.h>
+#include <linux/sched/user.h>
-struct user_struct;
struct cred;
struct inode;
#include <linux/rcupdate.h>
#include <linux/lockref.h>
#include <linux/stringhash.h>
+#include <linux/wait.h>
struct path;
struct vfsmount;
#define _LINUX_DELAYACCT_H
#include <uapi/linux/taskstats.h>
-#include <linux/sched.h>
-#include <linux/slab.h>
/*
* Per-task flags relevant to delay accounting
#define DELAYACCT_PF_BLKIO 0x00000002 /* I am waiting on IO */
#ifdef CONFIG_TASK_DELAY_ACCT
+struct task_delay_info {
+ spinlock_t lock;
+ unsigned int flags; /* Private per-task flags */
+
+ /* For each stat XXX, add following, aligned appropriately
+ *
+ * struct timespec XXX_start, XXX_end;
+ * u64 XXX_delay;
+ * u32 XXX_count;
+ *
+ * Atomicity of updates to XXX_delay, XXX_count protected by
+ * single lock above (split into XXX_lock if contention is an issue).
+ */
+
+ /*
+ * XXX_count is incremented on every XXX operation, the delay
+ * associated with the operation is added to XXX_delay.
+ * XXX_delay contains the accumulated delay time in nanoseconds.
+ */
+ u64 blkio_start; /* Shared by blkio, swapin */
+ u64 blkio_delay; /* wait for sync block io completion */
+ u64 swapin_delay; /* wait for swapin block io completion */
+ u32 blkio_count; /* total count of the number of sync block */
+ /* io operations performed */
+ u32 swapin_count; /* total count of the number of swapin block */
+ /* io operations performed */
+
+ u64 freepages_start;
+ u64 freepages_delay; /* wait for memory reclaim */
+ u32 freepages_count; /* total count of memory reclaim */
+};
+#endif
+#include <linux/sched.h>
+#include <linux/slab.h>
+
+#ifdef CONFIG_TASK_DELAY_ACCT
extern int delayacct_on; /* Delay accounting turned on/off */
extern struct kmem_cache *delayacct_cache;
extern void delayacct_init(void);
#include <linux/msi.h>
#include <linux/irqreturn.h>
#include <linux/rwsem.h>
-#include <linux/rcupdate.h>
+#include <linux/rculist.h>
struct acpi_dmar_header;
#include <linux/user.h>
#include <linux/bug.h>
+#include <linux/sched/task_stack.h>
+
#include <asm/elf.h>
#include <uapi/linux/elfcore.h>
#endif /* CONFIG_FAULT_INJECTION */
+struct kmem_cache;
+
#ifdef CONFIG_FAILSLAB
extern bool should_failslab(struct kmem_cache *s, gfp_t gfpflags);
#else
extern int vfs_rename(struct inode *, struct dentry *, struct inode *, struct dentry *, struct inode **, unsigned int);
extern int vfs_whiteout(struct inode *, struct dentry *);
+extern struct dentry *vfs_tmpfile(struct dentry *dentry, umode_t mode,
+ int open_flag);
+
/*
* VFS file helper functions.
*/
int (*rename) (struct inode *, struct dentry *,
struct inode *, struct dentry *, unsigned int);
int (*setattr) (struct dentry *, struct iattr *);
- int (*getattr) (struct vfsmount *mnt, struct dentry *, struct kstat *);
+ int (*getattr) (const struct path *, struct kstat *, u32, unsigned int);
ssize_t (*listxattr) (struct dentry *, char *, size_t);
int (*fiemap)(struct inode *, struct fiemap_extent_info *, u64 start,
u64 len);
int (*set_acl)(struct inode *, struct posix_acl *, int);
} ____cacheline_aligned;
+static inline ssize_t call_read_iter(struct file *file, struct kiocb *kio,
+ struct iov_iter *iter)
+{
+ return file->f_op->read_iter(kio, iter);
+}
+
+static inline ssize_t call_write_iter(struct file *file, struct kiocb *kio,
+ struct iov_iter *iter)
+{
+ return file->f_op->write_iter(kio, iter);
+}
+
+static inline int call_mmap(struct file *file, struct vm_area_struct *vma)
+{
+ return file->f_op->mmap(file, vma);
+}
+
+static inline int call_fsync(struct file *file, loff_t start, loff_t end,
+ int datasync)
+{
+ return file->f_op->fsync(file, start, end, datasync);
+}
+
ssize_t rw_copy_check_uvector(int type, const struct iovec __user * uvector,
unsigned long nr_segs, unsigned long fast_segs,
struct iovec *fast_pointer,
extern int vfs_dedupe_file_range(struct file *file,
struct file_dedupe_range *same);
-static inline int do_clone_file_range(struct file *file_in, loff_t pos_in,
- struct file *file_out, loff_t pos_out,
- u64 len)
-{
- int ret;
-
- sb_start_write(file_inode(file_out)->i_sb);
- ret = vfs_clone_file_range(file_in, pos_in, file_out, pos_out, len);
- sb_end_write(file_inode(file_out)->i_sb);
-
- return ret;
-}
-
struct super_operations {
struct inode *(*alloc_inode)(struct super_block *sb);
void (*destroy_inode)(struct inode *);
__sb_end_write(file_inode(file)->i_sb, SB_FREEZE_WRITE);
}
+static inline int do_clone_file_range(struct file *file_in, loff_t pos_in,
+ struct file *file_out, loff_t pos_out,
+ u64 len)
+{
+ int ret;
+
+ file_start_write(file_out);
+ ret = vfs_clone_file_range(file_in, pos_in, file_out, pos_out, len);
+ file_end_write(file_out);
+
+ return ret;
+}
+
/*
* get_write_access() gets write permission for a file.
* put_write_access() releases this write permission.
static inline const char *kernel_read_file_id_str(enum kernel_read_file_id id)
{
- if (id < 0 || id >= READING_MAX_ID)
+ if ((unsigned)id >= READING_MAX_ID)
return kernel_read_file_str[READING_UNKNOWN];
return kernel_read_file_str[id];
extern const struct inode_operations page_symlink_inode_operations;
extern void kfree_link(void *);
extern void generic_fillattr(struct inode *, struct kstat *);
-int vfs_getattr_nosec(struct path *path, struct kstat *stat);
-extern int vfs_getattr(struct path *, struct kstat *);
+extern int vfs_getattr_nosec(const struct path *, struct kstat *, u32, unsigned int);
+extern int vfs_getattr(const struct path *, struct kstat *, u32, unsigned int);
void __inode_add_bytes(struct inode *inode, loff_t bytes);
void inode_add_bytes(struct inode *inode, loff_t bytes);
void __inode_sub_bytes(struct inode *inode, loff_t bytes);
extern int iterate_dir(struct file *, struct dir_context *);
-extern int vfs_stat(const char __user *, struct kstat *);
-extern int vfs_lstat(const char __user *, struct kstat *);
-extern int vfs_fstat(unsigned int, struct kstat *);
-extern int vfs_fstatat(int , const char __user *, struct kstat *, int);
+extern int vfs_statx(int, const char __user *, int, struct kstat *, u32);
+extern int vfs_statx_fd(unsigned int, struct kstat *, u32, unsigned int);
+
+static inline int vfs_stat(const char __user *filename, struct kstat *stat)
+{
+ return vfs_statx(AT_FDCWD, filename, 0, stat, STATX_BASIC_STATS);
+}
+static inline int vfs_lstat(const char __user *name, struct kstat *stat)
+{
+ return vfs_statx(AT_FDCWD, name, AT_SYMLINK_NOFOLLOW,
+ stat, STATX_BASIC_STATS);
+}
+static inline int vfs_fstatat(int dfd, const char __user *filename,
+ struct kstat *stat, int flags)
+{
+ return vfs_statx(dfd, filename, flags, stat, STATX_BASIC_STATS);
+}
+static inline int vfs_fstat(int fd, struct kstat *stat)
+{
+ return vfs_statx_fd(fd, stat, STATX_BASIC_STATS, 0);
+}
+
+
extern const char *vfs_get_link(struct dentry *, struct delayed_call *);
extern int vfs_readlink(struct dentry *, char __user *, int);
extern loff_t dcache_dir_lseek(struct file *, loff_t, int);
extern int dcache_readdir(struct file *, struct dir_context *);
extern int simple_setattr(struct dentry *, struct iattr *);
-extern int simple_getattr(struct vfsmount *, struct dentry *, struct kstat *);
+extern int simple_getattr(const struct path *, struct kstat *, u32, unsigned int);
extern int simple_statfs(struct dentry *, struct kstatfs *);
extern int simple_open(struct inode *inode, struct file *file);
extern int simple_link(struct dentry *, struct inode *, struct dentry *);
};
#endif /* CONFIG_BLK_DEV_INTEGRITY */
-struct disk_devt {
- atomic_t count;
- void (*release)(struct disk_devt *disk_devt);
-};
-
-void put_disk_devt(struct disk_devt *disk_devt);
-void get_disk_devt(struct disk_devt *disk_devt);
struct gendisk {
/* major, first_minor and minors are input parameters only,
int first_minor;
int minors; /* maximum number of minors, =1 for
* disks that can't be partitioned. */
- struct disk_devt *disk_devt;
char disk_name[DISK_NAME_LEN]; /* name of major driver */
char *(*devnode)(struct gendisk *gd, umode_t *mode);
#include <linux/ktime.h>
#include <linux/init.h>
#include <linux/list.h>
-#include <linux/wait.h>
#include <linux/percpu.h>
#include <linux/timer.h>
#include <linux/timerqueue.h>
struct page *follow_huge_pud(struct mm_struct *mm, unsigned long address,
pud_t *pud, int flags);
int pmd_huge(pmd_t pmd);
-int pud_huge(pud_t pmd);
+int pud_huge(pud_t pud);
unsigned long hugetlb_change_protection(struct vm_area_struct *vma,
unsigned long address, unsigned long end, pgprot_t newprot);
#ifndef pgd_huge
#define pgd_huge(x) 0
#endif
+#ifndef p4d_huge
+#define p4d_huge(x) 0
+#endif
#ifndef pgd_write
static inline int pgd_write(pgd_t pgd)
#include <linux/device.h> /* for struct device */
#include <linux/sched.h> /* for completion */
#include <linux/mutex.h>
+#include <linux/rtmutex.h>
#include <linux/irqdomain.h> /* for Host Notify IRQ */
#include <linux/of.h> /* for struct device_node */
#include <linux/swab.h> /* for swab16 */
#include <linux/securebits.h>
#include <linux/seqlock.h>
#include <linux/rbtree.h>
+#include <linux/sched/autogroup.h>
#include <net/net_namespace.h>
#include <linux/sched/rt.h>
+#include <linux/mm_types.h>
#include <asm/thread_info.h>
extern struct cred init_cred;
-extern struct task_group root_task_group;
-
#ifdef CONFIG_CGROUP_SCHED
# define INIT_CGROUP_SCHED(tsk) \
.sched_task_group = &root_task_group,
#define ICC_IGRPEN0_EL1_MASK (1 << ICC_IGRPEN0_EL1_SHIFT)
#define ICC_IGRPEN1_EL1_SHIFT 0
#define ICC_IGRPEN1_EL1_MASK (1 << ICC_IGRPEN1_EL1_SHIFT)
+#define ICC_SRE_EL1_DIB (1U << 2)
+#define ICC_SRE_EL1_DFB (1U << 1)
#define ICC_SRE_EL1_SRE (1U << 0)
/*
{
return NULL;
}
+static inline bool irq_domain_check_msi_remap(void)
+{
+ return false;
+}
#endif /* !CONFIG_IRQ_DOMAIN */
#endif /* _LINUX_IRQDOMAIN_H */
struct static_key {
atomic_t enabled;
/*
+ * Note:
+ * To make anonymous unions work with old compilers, the static
+ * initialization of them requires brackets. This creates a dependency
+ * on the order of the struct with the initializers. If any fields
+ * are added, STATIC_KEY_INIT_TRUE and STATIC_KEY_INIT_FALSE may need
+ * to be modified.
+ *
* bit 0 => 1 if key is initially true
* 0 if initially false
* bit 1 => 1 if points to struct static_key_mod
*/
#define STATIC_KEY_INIT_TRUE \
{ .enabled = { 1 }, \
- .entries = (void *)JUMP_TYPE_TRUE }
+ { .entries = (void *)JUMP_TYPE_TRUE } }
#define STATIC_KEY_INIT_FALSE \
{ .enabled = { 0 }, \
- .entries = (void *)JUMP_TYPE_FALSE }
+ { .entries = (void *)JUMP_TYPE_FALSE } }
#else /* !HAVE_JUMP_LABEL */
#ifndef _LINUX_KASAN_H
#define _LINUX_KASAN_H
-#include <linux/sched.h>
#include <linux/types.h>
struct kmem_cache;
extern pte_t kasan_zero_pte[PTRS_PER_PTE];
extern pmd_t kasan_zero_pmd[PTRS_PER_PMD];
extern pud_t kasan_zero_pud[PTRS_PER_PUD];
+extern p4d_t kasan_zero_p4d[PTRS_PER_P4D];
void kasan_populate_zero_shadow(const void *shadow_start,
const void *shadow_end);
}
/* Enable reporting bugs after kasan_disable_current() */
-static inline void kasan_enable_current(void)
-{
- current->kasan_depth++;
-}
+extern void kasan_enable_current(void);
/* Disable reporting bugs for current task */
-static inline void kasan_disable_current(void)
-{
- current->kasan_depth--;
-}
+extern void kasan_disable_current(void);
void kasan_unpoison_shadow(const void *address, size_t size);
!test_bit(KEY_FLAG_NEGATIVE, &key->flags);
}
-#define rcu_dereference_key(KEY) \
+#define dereference_key_rcu(KEY) \
+ (rcu_dereference((KEY)->payload.rcu_data0))
+
+#define dereference_key_locked(KEY) \
(rcu_dereference_protected((KEY)->payload.rcu_data0, \
rwsem_is_locked(&((struct key *)(KEY))->sem)))
#ifndef _LINUX_KHUGEPAGED_H
#define _LINUX_KHUGEPAGED_H
-#include <linux/sched.h> /* MMF_VM_HUGEPAGE */
+#include <linux/sched/coredump.h> /* MMF_VM_HUGEPAGE */
+
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
extern struct attribute_group khugepaged_attr_group;
#include <linux/pagemap.h>
#include <linux/rmap.h>
#include <linux/sched.h>
+#include <linux/sched/coredump.h>
struct stable_node;
struct mem_cgroup;
#include <linux/context_tracking.h>
#include <linux/irqbypass.h>
#include <linux/swait.h>
+#include <linux/refcount.h>
#include <asm/signal.h>
#include <linux/kvm.h>
#endif
struct kvm_vm_stat stat;
struct kvm_arch arch;
- atomic_t users_count;
+ refcount_t users_count;
#ifdef KVM_COALESCED_MMIO_PAGE_OFFSET
struct kvm_coalesced_mmio_ring *coalesced_mmio_ring;
spinlock_t ring_lock;
struct nd_interleave_set {
u64 cookie;
+ /* compatibility with initial buggy Linux implementation */
+ u64 altcookie;
};
struct nd_mapping_desc {
#include <linux/types.h>
#include <linux/bitops.h>
-/*
- * deal with unrepresentable constant logarithms
- */
-extern __attribute__((const, noreturn))
-int ____ilog2_NaN(void);
-
/*
* non-constant log of base 2 calculators
* - the arch may override these in asm/bitops.h if they can be implemented
#define ilog2(n) \
( \
__builtin_constant_p(n) ? ( \
- (n) < 1 ? ____ilog2_NaN() : \
+ (n) < 2 ? 0 : \
(n) & (1ULL << 63) ? 63 : \
(n) & (1ULL << 62) ? 62 : \
(n) & (1ULL << 61) ? 61 : \
(n) & (1ULL << 4) ? 4 : \
(n) & (1ULL << 3) ? 3 : \
(n) & (1ULL << 2) ? 2 : \
- (n) & (1ULL << 1) ? 1 : \
- (n) & (1ULL << 0) ? 0 : \
- ____ilog2_NaN() \
- ) : \
+ 1 ) : \
(sizeof(n) <= 4) ? \
__ilog2_u32(n) : \
__ilog2_u64(n) \
int i;
for (i = ETH_ALEN; i > 0; i--) {
- addr[i - 1] = mac && 0xFF;
+ addr[i - 1] = mac & 0xFF;
mac >>= 8;
}
}
return ptep;
}
+#ifdef __PAGETABLE_P4D_FOLDED
+static inline int __p4d_alloc(struct mm_struct *mm, pgd_t *pgd,
+ unsigned long address)
+{
+ return 0;
+}
+#else
+int __p4d_alloc(struct mm_struct *mm, pgd_t *pgd, unsigned long address);
+#endif
+
#ifdef __PAGETABLE_PUD_FOLDED
-static inline int __pud_alloc(struct mm_struct *mm, pgd_t *pgd,
+static inline int __pud_alloc(struct mm_struct *mm, p4d_t *p4d,
unsigned long address)
{
return 0;
}
#else
-int __pud_alloc(struct mm_struct *mm, pgd_t *pgd, unsigned long address);
+int __pud_alloc(struct mm_struct *mm, p4d_t *p4d, unsigned long address);
#endif
#if defined(__PAGETABLE_PMD_FOLDED) || !defined(CONFIG_MMU)
* Remove it when 4level-fixup.h has been removed.
*/
#if defined(CONFIG_MMU) && !defined(__ARCH_HAS_4LEVEL_HACK)
-static inline pud_t *pud_alloc(struct mm_struct *mm, pgd_t *pgd, unsigned long address)
+
+#ifndef __ARCH_HAS_5LEVEL_HACK
+static inline p4d_t *p4d_alloc(struct mm_struct *mm, pgd_t *pgd,
+ unsigned long address)
+{
+ return (unlikely(pgd_none(*pgd)) && __p4d_alloc(mm, pgd, address)) ?
+ NULL : p4d_offset(pgd, address);
+}
+
+static inline pud_t *pud_alloc(struct mm_struct *mm, p4d_t *p4d,
+ unsigned long address)
{
- return (unlikely(pgd_none(*pgd)) && __pud_alloc(mm, pgd, address))?
- NULL: pud_offset(pgd, address);
+ return (unlikely(p4d_none(*p4d)) && __pud_alloc(mm, p4d, address)) ?
+ NULL : pud_offset(p4d, address);
}
+#endif /* !__ARCH_HAS_5LEVEL_HACK */
static inline pmd_t *pmd_alloc(struct mm_struct *mm, pud_t *pud, unsigned long address)
{
struct page *sparse_mem_map_populate(unsigned long pnum, int nid);
pgd_t *vmemmap_pgd_populate(unsigned long addr, int node);
-pud_t *vmemmap_pud_populate(pgd_t *pgd, unsigned long addr, int node);
+p4d_t *vmemmap_p4d_populate(pgd_t *pgd, unsigned long addr, int node);
+pud_t *vmemmap_pud_populate(p4d_t *p4d, unsigned long addr, int node);
pmd_t *vmemmap_pmd_populate(pud_t *pud, unsigned long addr, int node);
pte_t *vmemmap_pte_populate(pmd_t *pmd, unsigned long addr, int node);
void *vmemmap_alloc_block(unsigned long size, int node);
#ifndef _LINUX_MM_TYPES_H
#define _LINUX_MM_TYPES_H
+#include <linux/mm_types_task.h>
+
#include <linux/auxvec.h>
-#include <linux/types.h>
-#include <linux/threads.h>
#include <linux/list.h>
#include <linux/spinlock.h>
#include <linux/rbtree.h>
#include <linux/uprobes.h>
#include <linux/page-flags-layout.h>
#include <linux/workqueue.h>
-#include <asm/page.h>
+
#include <asm/mmu.h>
#ifndef AT_VECTOR_SIZE_ARCH
struct address_space;
struct mem_cgroup;
-#define USE_SPLIT_PTE_PTLOCKS (NR_CPUS >= CONFIG_SPLIT_PTLOCK_CPUS)
-#define USE_SPLIT_PMD_PTLOCKS (USE_SPLIT_PTE_PTLOCKS && \
- IS_ENABLED(CONFIG_ARCH_ENABLE_SPLIT_PMD_PTLOCK))
-#define ALLOC_SPLIT_PTLOCKS (SPINLOCK_SIZE > BITS_PER_LONG/8)
-
/*
* Each physical page in the system has a struct page associated with
* it to keep track of whatever it is we are using the page for at the
#endif
;
-struct page_frag {
- struct page *page;
-#if (BITS_PER_LONG > 32) || (PAGE_SIZE >= 65536)
- __u32 offset;
- __u32 size;
-#else
- __u16 offset;
- __u16 size;
-#endif
-};
-
#define PAGE_FRAG_CACHE_MAX_SIZE __ALIGN_MASK(32768, ~PAGE_MASK)
#define PAGE_FRAG_CACHE_MAX_ORDER get_order(PAGE_FRAG_CACHE_MAX_SIZE)
struct completion startup;
};
-enum {
- MM_FILEPAGES, /* Resident file mapping pages */
- MM_ANONPAGES, /* Resident anonymous pages */
- MM_SWAPENTS, /* Anonymous swap entries */
- MM_SHMEMPAGES, /* Resident shared memory pages */
- NR_MM_COUNTERS
-};
-
-#if USE_SPLIT_PTE_PTLOCKS && defined(CONFIG_MMU)
-#define SPLIT_RSS_COUNTING
-/* per-thread cached information, */
-struct task_rss_stat {
- int events; /* for synchronization threshold */
- int count[NR_MM_COUNTERS];
-};
-#endif /* USE_SPLIT_PTE_PTLOCKS */
-
-struct mm_rss_stat {
- atomic_long_t count[NR_MM_COUNTERS];
-};
-
struct kioctx_table;
struct mm_struct {
struct vm_area_struct *mmap; /* list of VMAs */
struct work_struct async_put_work;
};
+extern struct mm_struct init_mm;
+
static inline void mm_init_cpumask(struct mm_struct *mm)
{
#ifdef CONFIG_CPUMASK_OFFSTACK
--- /dev/null
+#ifndef _LINUX_MM_TYPES_TASK_H
+#define _LINUX_MM_TYPES_TASK_H
+
+/*
+ * Here are the definitions of the MM data types that are embedded in 'struct task_struct'.
+ *
+ * (These are defined separately to decouple sched.h from mm_types.h as much as possible.)
+ */
+
+#include <linux/types.h>
+#include <linux/threads.h>
+#include <linux/atomic.h>
+#include <linux/cpumask.h>
+
+#include <asm/page.h>
+
+#define USE_SPLIT_PTE_PTLOCKS (NR_CPUS >= CONFIG_SPLIT_PTLOCK_CPUS)
+#define USE_SPLIT_PMD_PTLOCKS (USE_SPLIT_PTE_PTLOCKS && \
+ IS_ENABLED(CONFIG_ARCH_ENABLE_SPLIT_PMD_PTLOCK))
+#define ALLOC_SPLIT_PTLOCKS (SPINLOCK_SIZE > BITS_PER_LONG/8)
+
+/*
+ * The per task VMA cache array:
+ */
+#define VMACACHE_BITS 2
+#define VMACACHE_SIZE (1U << VMACACHE_BITS)
+#define VMACACHE_MASK (VMACACHE_SIZE - 1)
+
+struct vmacache {
+ u32 seqnum;
+ struct vm_area_struct *vmas[VMACACHE_SIZE];
+};
+
+enum {
+ MM_FILEPAGES, /* Resident file mapping pages */
+ MM_ANONPAGES, /* Resident anonymous pages */
+ MM_SWAPENTS, /* Anonymous swap entries */
+ MM_SHMEMPAGES, /* Resident shared memory pages */
+ NR_MM_COUNTERS
+};
+
+#if USE_SPLIT_PTE_PTLOCKS && defined(CONFIG_MMU)
+#define SPLIT_RSS_COUNTING
+/* per-thread cached information, */
+struct task_rss_stat {
+ int events; /* for synchronization threshold */
+ int count[NR_MM_COUNTERS];
+};
+#endif /* USE_SPLIT_PTE_PTLOCKS */
+
+struct mm_rss_stat {
+ atomic_long_t count[NR_MM_COUNTERS];
+};
+
+struct page_frag {
+ struct page *page;
+#if (BITS_PER_LONG > 32) || (PAGE_SIZE >= 65536)
+ __u32 offset;
+ __u32 size;
+#else
+ __u16 offset;
+ __u16 size;
+#endif
+};
+
+/* Track pages that require TLB flushes */
+struct tlbflush_unmap_batch {
+#ifdef CONFIG_ARCH_WANT_BATCHED_UNMAP_TLB_FLUSH
+ /*
+ * Each bit set is a CPU that potentially has a TLB entry for one of
+ * the PFNs being flushed. See set_tlb_ubc_flush_pending().
+ */
+ struct cpumask cpumask;
+
+ /* True if any bit in cpumask is set */
+ bool flush_required;
+
+ /*
+ * If true then the PTE was dirty when unmapped. The entry must be
+ * flushed before IO is initiated or a stale TLB entry potentially
+ * allows an update without redirtying the page.
+ */
+ bool writable;
+#endif
+};
+
+#endif /* _LINUX_MM_TYPES_TASK_H */
enum {
NAPI_STATE_SCHED, /* Poll is scheduled */
+ NAPI_STATE_MISSED, /* reschedule a napi */
NAPI_STATE_DISABLE, /* Disable pending */
NAPI_STATE_NPSVC, /* Netpoll - don't dequeue from poll_list */
NAPI_STATE_HASHED, /* In NAPI hash (busy polling possible) */
};
enum {
- NAPIF_STATE_SCHED = (1UL << NAPI_STATE_SCHED),
- NAPIF_STATE_DISABLE = (1UL << NAPI_STATE_DISABLE),
- NAPIF_STATE_NPSVC = (1UL << NAPI_STATE_NPSVC),
- NAPIF_STATE_HASHED = (1UL << NAPI_STATE_HASHED),
- NAPIF_STATE_NO_BUSY_POLL = (1UL << NAPI_STATE_NO_BUSY_POLL),
- NAPIF_STATE_IN_BUSY_POLL = (1UL << NAPI_STATE_IN_BUSY_POLL),
+ NAPIF_STATE_SCHED = BIT(NAPI_STATE_SCHED),
+ NAPIF_STATE_MISSED = BIT(NAPI_STATE_MISSED),
+ NAPIF_STATE_DISABLE = BIT(NAPI_STATE_DISABLE),
+ NAPIF_STATE_NPSVC = BIT(NAPI_STATE_NPSVC),
+ NAPIF_STATE_HASHED = BIT(NAPI_STATE_HASHED),
+ NAPIF_STATE_NO_BUSY_POLL = BIT(NAPI_STATE_NO_BUSY_POLL),
+ NAPIF_STATE_IN_BUSY_POLL = BIT(NAPI_STATE_IN_BUSY_POLL),
};
enum gro_result {
return test_bit(NAPI_STATE_DISABLE, &n->state);
}
-/**
- * napi_schedule_prep - check if NAPI can be scheduled
- * @n: NAPI context
- *
- * Test if NAPI routine is already running, and if not mark
- * it as running. This is used as a condition variable to
- * insure only one NAPI poll instance runs. We also make
- * sure there is no pending NAPI disable.
- */
-static inline bool napi_schedule_prep(struct napi_struct *n)
-{
- return !napi_disable_pending(n) &&
- !test_and_set_bit(NAPI_STATE_SCHED, &n->state);
-}
+bool napi_schedule_prep(struct napi_struct *n);
/**
* napi_schedule - schedule NAPI poll
extern int nfs_post_op_update_inode(struct inode *inode, struct nfs_fattr *fattr);
extern int nfs_post_op_update_inode_force_wcc(struct inode *inode, struct nfs_fattr *fattr);
extern int nfs_post_op_update_inode_force_wcc_locked(struct inode *inode, struct nfs_fattr *fattr);
-extern int nfs_getattr(struct vfsmount *, struct dentry *, struct kstat *);
+extern int nfs_getattr(const struct path *, struct kstat *, u32, unsigned int);
extern void nfs_access_add_cache(struct inode *, struct nfs_access_entry *);
extern void nfs_access_set_mask(struct nfs_access_entry *, u32);
extern int nfs_permission(struct inode *, int);
#include <linux/sched.h>
#include <asm/irq.h>
+#ifdef CONFIG_LOCKUP_DETECTOR
+extern void touch_softlockup_watchdog_sched(void);
+extern void touch_softlockup_watchdog(void);
+extern void touch_softlockup_watchdog_sync(void);
+extern void touch_all_softlockup_watchdogs(void);
+extern int proc_dowatchdog_thresh(struct ctl_table *table, int write,
+ void __user *buffer,
+ size_t *lenp, loff_t *ppos);
+extern unsigned int softlockup_panic;
+extern unsigned int hardlockup_panic;
+void lockup_detector_init(void);
+#else
+static inline void touch_softlockup_watchdog_sched(void)
+{
+}
+static inline void touch_softlockup_watchdog(void)
+{
+}
+static inline void touch_softlockup_watchdog_sync(void)
+{
+}
+static inline void touch_all_softlockup_watchdogs(void)
+{
+}
+static inline void lockup_detector_init(void)
+{
+}
+#endif
+
+#ifdef CONFIG_DETECT_HUNG_TASK
+void reset_hung_task_detector(void);
+#else
+static inline void reset_hung_task_detector(void)
+{
+}
+#endif
+
/*
* The run state of the lockup detectors is controlled by the content of the
* 'watchdog_enabled' variable. Each lockup detector has its dedicated bit -
#define __INCLUDE_LINUX_OOM_H
-#include <linux/sched.h>
+#include <linux/sched/signal.h>
#include <linux/types.h>
#include <linux/nodemask.h>
#include <uapi/linux/oom.h>
void pci_free_irq_vectors(struct pci_dev *dev);
int pci_irq_vector(struct pci_dev *dev, unsigned int nr);
const struct cpumask *pci_irq_get_affinity(struct pci_dev *pdev, int vec);
+int pci_irq_get_node(struct pci_dev *pdev, int vec);
#else
static inline int pci_msi_vec_count(struct pci_dev *dev) { return -ENOSYS; }
{
return cpu_possible_mask;
}
+
+static inline int pci_irq_get_node(struct pci_dev *pdev, int vec)
+{
+ return first_online_node;
+}
#endif
static inline int
#ifndef _LINUX_PERF_REGS_H
#define _LINUX_PERF_REGS_H
+#include <linux/sched/task_stack.h>
+
struct perf_regs {
__u64 abi;
struct pt_regs *regs;
#ifndef _LINUX_PID_H
#define _LINUX_PID_H
-#include <linux/rcupdate.h>
+#include <linux/rculist.h>
enum pid_type
{
struct notifier_block *notifier);
int dev_pm_qos_remove_notifier(struct device *dev,
struct notifier_block *notifier);
-int dev_pm_qos_add_global_notifier(struct notifier_block *notifier);
-int dev_pm_qos_remove_global_notifier(struct notifier_block *notifier);
void dev_pm_qos_constraints_init(struct device *dev);
void dev_pm_qos_constraints_destroy(struct device *dev);
int dev_pm_qos_add_ancestor_request(struct device *dev,
{
return dev->power.qos->flags_req->data.flr.flags;
}
+
+static inline s32 dev_pm_qos_raw_read_value(struct device *dev)
+{
+ return IS_ERR_OR_NULL(dev->power.qos) ?
+ 0 : pm_qos_read_value(&dev->power.qos->resume_latency);
+}
#else
static inline enum pm_qos_flags_status __dev_pm_qos_flags(struct device *dev,
s32 mask)
static inline int dev_pm_qos_remove_notifier(struct device *dev,
struct notifier_block *notifier)
{ return 0; }
-static inline int dev_pm_qos_add_global_notifier(
- struct notifier_block *notifier)
- { return 0; }
-static inline int dev_pm_qos_remove_global_notifier(
- struct notifier_block *notifier)
- { return 0; }
static inline void dev_pm_qos_constraints_init(struct device *dev)
{
dev->power.power_state = PMSG_ON;
static inline s32 dev_pm_qos_requested_resume_latency(struct device *dev) { return 0; }
static inline s32 dev_pm_qos_requested_flags(struct device *dev) { return 0; }
+static inline s32 dev_pm_qos_raw_read_value(struct device *dev) { return 0; }
#endif
#endif
/* We use the MSB mostly because its available */
#define PREEMPT_NEED_RESCHED 0x80000000
+#define PREEMPT_DISABLED (PREEMPT_DISABLE_OFFSET + PREEMPT_ENABLED)
+
+/*
+ * Disable preemption until the scheduler is running -- use an unconditional
+ * value so that it also works on !PREEMPT_COUNT kernels.
+ *
+ * Reset by start_kernel()->sched_init()->init_idle()->init_idle_preempt_count().
+ */
+#define INIT_PREEMPT_COUNT PREEMPT_OFFSET
+
+/*
+ * Initial preempt_count value; reflects the preempt_count schedule invariant
+ * which states that during context switches:
+ *
+ * preempt_count() == 2*PREEMPT_DISABLE_OFFSET
+ *
+ * Note: PREEMPT_DISABLE_OFFSET is 0 for !PREEMPT_COUNT kernels.
+ * Note: See finish_task_switch().
+ */
+#define FORK_PREEMPT_COUNT (2*PREEMPT_DISABLE_OFFSET + PREEMPT_ENABLED)
+
/* preempt_count() and related functions, depends on PREEMPT_NEED_RESCHED */
#include <asm/preempt.h>
#include <linux/compiler.h> /* For unlikely. */
#include <linux/sched.h> /* For struct task_struct. */
+#include <linux/sched/signal.h> /* For send_sig(), same_thread_group(), etc. */
#include <linux/err.h> /* for IS_ERR_VALUE */
#include <linux/bug.h> /* For BUG_ON. */
#include <linux/pid_namespace.h> /* For task_active_pid_ns. */
--- /dev/null
+#ifndef _LINUX_PURGATORY_H
+#define _LINUX_PURGATORY_H
+
+#include <linux/types.h>
+#include <crypto/sha.h>
+#include <uapi/linux/kexec.h>
+
+struct kexec_sha_region {
+ unsigned long start;
+ unsigned long len;
+};
+
+/*
+ * These forward declarations serve two purposes:
+ *
+ * 1) Make sparse happy when checking arch/purgatory
+ * 2) Document that these are required to be global so the symbol
+ * lookup in kexec works
+ */
+extern struct kexec_sha_region purgatory_sha_regions[KEXEC_SEGMENT_MAX];
+extern u8 purgatory_sha256_digest[SHA256_DIGEST_SIZE];
+
+#endif
extern int add_random_ready_callback(struct random_ready_callback *rdy);
extern void del_random_ready_callback(struct random_ready_callback *rdy);
extern void get_random_bytes_arch(void *buf, int nbytes);
-extern int random_int_secret_init(void);
#ifndef MODULE
extern const struct file_operations random_fops, urandom_fops;
#endif
-unsigned int get_random_int(void);
-unsigned long get_random_long(void);
+u32 get_random_u32(void);
+u64 get_random_u64(void);
+static inline unsigned int get_random_int(void)
+{
+ return get_random_u32();
+}
+static inline unsigned long get_random_long(void)
+{
+#if BITS_PER_LONG == 64
+ return get_random_u64();
+#else
+ return get_random_u32();
+#endif
+}
+
unsigned long randomize_page(unsigned long start, unsigned long range);
u32 prandom_u32(void);
#include <linux/cpumask.h>
#include <linux/seqlock.h>
#include <linux/lockdep.h>
-#include <linux/completion.h>
#include <linux/debugobjects.h>
#include <linux/bug.h>
#include <linux/compiler.h>
void synchronize_sched(void);
-/*
- * Structure allowing asynchronous waiting on RCU.
- */
-struct rcu_synchronize {
- struct rcu_head head;
- struct completion completion;
-};
-void wakeme_after_rcu(struct rcu_head *head);
-
-void __wait_rcu_gp(bool checktiny, int n, call_rcu_func_t *crcu_array,
- struct rcu_synchronize *rs_array);
-
-#define _wait_rcu_gp(checktiny, ...) \
-do { \
- call_rcu_func_t __crcu_array[] = { __VA_ARGS__ }; \
- struct rcu_synchronize __rs_array[ARRAY_SIZE(__crcu_array)]; \
- __wait_rcu_gp(checktiny, ARRAY_SIZE(__crcu_array), \
- __crcu_array, __rs_array); \
-} while (0)
-
-#define wait_rcu_gp(...) _wait_rcu_gp(false, __VA_ARGS__)
-
-/**
- * synchronize_rcu_mult - Wait concurrently for multiple grace periods
- * @...: List of call_rcu() functions for the flavors to wait on.
- *
- * This macro waits concurrently for multiple flavors of RCU grace periods.
- * For example, synchronize_rcu_mult(call_rcu, call_rcu_bh) would wait
- * on concurrent RCU and RCU-bh grace periods. Waiting on a give SRCU
- * domain requires you to write a wrapper function for that SRCU domain's
- * call_srcu() function, supplying the corresponding srcu_struct.
- *
- * If Tiny RCU, tell _wait_rcu_gp() not to bother waiting for RCU
- * or RCU-bh, given that anywhere synchronize_rcu_mult() can be called
- * is automatically a grace period.
- */
-#define synchronize_rcu_mult(...) \
- _wait_rcu_gp(IS_ENABLED(CONFIG_TINY_RCU), __VA_ARGS__)
-
/**
* call_rcu_tasks() - Queue an RCU for invocation task-based grace period
* @head: structure to be used for queueing the RCU updates.
--- /dev/null
+#ifndef _LINUX_SCHED_RCUPDATE_WAIT_H
+#define _LINUX_SCHED_RCUPDATE_WAIT_H
+
+/*
+ * RCU synchronization types and methods:
+ */
+
+#include <linux/rcupdate.h>
+#include <linux/completion.h>
+
+/*
+ * Structure allowing asynchronous waiting on RCU.
+ */
+struct rcu_synchronize {
+ struct rcu_head head;
+ struct completion completion;
+};
+void wakeme_after_rcu(struct rcu_head *head);
+
+void __wait_rcu_gp(bool checktiny, int n, call_rcu_func_t *crcu_array,
+ struct rcu_synchronize *rs_array);
+
+#define _wait_rcu_gp(checktiny, ...) \
+do { \
+ call_rcu_func_t __crcu_array[] = { __VA_ARGS__ }; \
+ struct rcu_synchronize __rs_array[ARRAY_SIZE(__crcu_array)]; \
+ __wait_rcu_gp(checktiny, ARRAY_SIZE(__crcu_array), \
+ __crcu_array, __rs_array); \
+} while (0)
+
+#define wait_rcu_gp(...) _wait_rcu_gp(false, __VA_ARGS__)
+
+/**
+ * synchronize_rcu_mult - Wait concurrently for multiple grace periods
+ * @...: List of call_rcu() functions for the flavors to wait on.
+ *
+ * This macro waits concurrently for multiple flavors of RCU grace periods.
+ * For example, synchronize_rcu_mult(call_rcu, call_rcu_bh) would wait
+ * on concurrent RCU and RCU-bh grace periods. Waiting on a give SRCU
+ * domain requires you to write a wrapper function for that SRCU domain's
+ * call_srcu() function, supplying the corresponding srcu_struct.
+ *
+ * If Tiny RCU, tell _wait_rcu_gp() not to bother waiting for RCU
+ * or RCU-bh, given that anywhere synchronize_rcu_mult() can be called
+ * is automatically a grace period.
+ */
+#define synchronize_rcu_mult(...) \
+ _wait_rcu_gp(IS_ENABLED(CONFIG_TINY_RCU), __VA_ARGS__)
+
+#endif /* _LINUX_SCHED_RCUPDATE_WAIT_H */
might_sleep();
}
-static inline void rcu_barrier_bh(void)
-{
- wait_rcu_gp(call_rcu_bh);
-}
-
-static inline void rcu_barrier_sched(void)
-{
- wait_rcu_gp(call_rcu_sched);
-}
+extern void rcu_barrier_bh(void);
+extern void rcu_barrier_sched(void);
static inline void synchronize_rcu_expedited(void)
{
int uV; /* suspend voltage */
unsigned int mode; /* suspend regulator operating mode */
int enabled; /* is regulator enabled in this suspend state */
- int disabled; /* is the regulator disbled in this suspend state */
+ int disabled; /* is the regulator disabled in this suspend state */
};
/**
#include <linux/list_nulls.h>
#include <linux/workqueue.h>
#include <linux/mutex.h>
-#include <linux/rcupdate.h>
+#include <linux/rculist.h>
/*
* The end of the chain is marked with a special nulls marks which has
#ifndef _LINUX_SCHED_H
#define _LINUX_SCHED_H
-#include <uapi/linux/sched.h>
-
-#include <linux/sched/prio.h>
-
-
-struct sched_param {
- int sched_priority;
-};
-
-#include <asm/param.h> /* for HZ */
+/*
+ * Define 'struct task_struct' and provide the main scheduler
+ * APIs (schedule(), wakeup variants, etc.)
+ */
-#include <linux/capability.h>
-#include <linux/threads.h>
-#include <linux/kernel.h>
-#include <linux/types.h>
-#include <linux/timex.h>
-#include <linux/jiffies.h>
-#include <linux/plist.h>
-#include <linux/rbtree.h>
-#include <linux/thread_info.h>
-#include <linux/cpumask.h>
-#include <linux/errno.h>
-#include <linux/nodemask.h>
-#include <linux/mm_types.h>
-#include <linux/preempt.h>
+#include <uapi/linux/sched.h>
-#include <asm/page.h>
-#include <asm/ptrace.h>
+#include <asm/current.h>
-#include <linux/smp.h>
+#include <linux/pid.h>
#include <linux/sem.h>
#include <linux/shm.h>
-#include <linux/signal.h>
-#include <linux/compiler.h>
-#include <linux/completion.h>
-#include <linux/pid.h>
-#include <linux/percpu.h>
-#include <linux/topology.h>
+#include <linux/kcov.h>
+#include <linux/mutex.h>
+#include <linux/plist.h>
+#include <linux/hrtimer.h>
#include <linux/seccomp.h>
+#include <linux/nodemask.h>
#include <linux/rcupdate.h>
-#include <linux/rculist.h>
-#include <linux/rtmutex.h>
-
-#include <linux/time.h>
-#include <linux/param.h>
#include <linux/resource.h>
-#include <linux/timer.h>
-#include <linux/hrtimer.h>
-#include <linux/kcov.h>
-#include <linux/task_io_accounting.h>
#include <linux/latencytop.h>
-#include <linux/cred.h>
-#include <linux/llist.h>
-#include <linux/uidgid.h>
-#include <linux/gfp.h>
-#include <linux/magic.h>
-#include <linux/cgroup-defs.h>
-
-#include <asm/processor.h>
-
-#define SCHED_ATTR_SIZE_VER0 48 /* sizeof first published struct */
-
-/*
- * Extended scheduling parameters data structure.
- *
- * This is needed because the original struct sched_param can not be
- * altered without introducing ABI issues with legacy applications
- * (e.g., in sched_getparam()).
- *
- * However, the possibility of specifying more than just a priority for
- * the tasks may be useful for a wide variety of application fields, e.g.,
- * multimedia, streaming, automation and control, and many others.
- *
- * This variant (sched_attr) is meant at describing a so-called
- * sporadic time-constrained task. In such model a task is specified by:
- * - the activation period or minimum instance inter-arrival time;
- * - the maximum (or average, depending on the actual scheduling
- * discipline) computation time of all instances, a.k.a. runtime;
- * - the deadline (relative to the actual activation time) of each
- * instance.
- * Very briefly, a periodic (sporadic) task asks for the execution of
- * some specific computation --which is typically called an instance--
- * (at most) every period. Moreover, each instance typically lasts no more
- * than the runtime and must be completed by time instant t equal to
- * the instance activation time + the deadline.
- *
- * This is reflected by the actual fields of the sched_attr structure:
- *
- * @size size of the structure, for fwd/bwd compat.
- *
- * @sched_policy task's scheduling policy
- * @sched_flags for customizing the scheduler behaviour
- * @sched_nice task's nice value (SCHED_NORMAL/BATCH)
- * @sched_priority task's static priority (SCHED_FIFO/RR)
- * @sched_deadline representative of the task's deadline
- * @sched_runtime representative of the task's runtime
- * @sched_period representative of the task's period
- *
- * Given this task model, there are a multiplicity of scheduling algorithms
- * and policies, that can be used to ensure all the tasks will make their
- * timing constraints.
- *
- * As of now, the SCHED_DEADLINE policy (sched_dl scheduling class) is the
- * only user of this new interface. More information about the algorithm
- * available in the scheduling class file or in Documentation/.
- */
-struct sched_attr {
- u32 size;
-
- u32 sched_policy;
- u64 sched_flags;
-
- /* SCHED_NORMAL, SCHED_BATCH */
- s32 sched_nice;
-
- /* SCHED_FIFO, SCHED_RR */
- u32 sched_priority;
-
- /* SCHED_DEADLINE */
- u64 sched_runtime;
- u64 sched_deadline;
- u64 sched_period;
-};
+#include <linux/sched/prio.h>
+#include <linux/signal_types.h>
+#include <linux/mm_types_task.h>
+#include <linux/task_io_accounting.h>
-struct futex_pi_state;
-struct robust_list_head;
+/* task_struct member predeclarations (sorted alphabetically): */
+struct audit_context;
+struct backing_dev_info;
struct bio_list;
-struct fs_struct;
-struct perf_event_context;
struct blk_plug;
-struct filename;
+struct cfs_rq;
+struct fs_struct;
+struct futex_pi_state;
+struct io_context;
+struct mempolicy;
struct nameidata;
-
-#define VMACACHE_BITS 2
-#define VMACACHE_SIZE (1U << VMACACHE_BITS)
-#define VMACACHE_MASK (VMACACHE_SIZE - 1)
-
-/*
- * These are the constant used to fake the fixed-point load-average
- * counting. Some notes:
- * - 11 bit fractions expand to 22 bits by the multiplies: this gives
- * a load-average precision of 10 bits integer + 11 bits fractional
- * - if you want to count load-averages more often, you need more
- * precision, or rounding will get you. With 2-second counting freq,
- * the EXP_n values would be 1981, 2034 and 2043 if still using only
- * 11 bit fractions.
- */
-extern unsigned long avenrun[]; /* Load averages */
-extern void get_avenrun(unsigned long *loads, unsigned long offset, int shift);
-
-#define FSHIFT 11 /* nr of bits of precision */
-#define FIXED_1 (1<<FSHIFT) /* 1.0 as fixed-point */
-#define LOAD_FREQ (5*HZ+1) /* 5 sec intervals */
-#define EXP_1 1884 /* 1/exp(5sec/1min) as fixed-point */
-#define EXP_5 2014 /* 1/exp(5sec/5min) */
-#define EXP_15 2037 /* 1/exp(5sec/15min) */
-
-#define CALC_LOAD(load,exp,n) \
- load *= exp; \
- load += n*(FIXED_1-exp); \
- load >>= FSHIFT;
-
-extern unsigned long total_forks;
-extern int nr_threads;
-DECLARE_PER_CPU(unsigned long, process_counts);
-extern int nr_processes(void);
-extern unsigned long nr_running(void);
-extern bool single_task_running(void);
-extern unsigned long nr_iowait(void);
-extern unsigned long nr_iowait_cpu(int cpu);
-extern void get_iowait_load(unsigned long *nr_waiters, unsigned long *load);
-
-extern void calc_global_load(unsigned long ticks);
-
-#if defined(CONFIG_SMP) && defined(CONFIG_NO_HZ_COMMON)
-extern void cpu_load_update_nohz_start(void);
-extern void cpu_load_update_nohz_stop(void);
-#else
-static inline void cpu_load_update_nohz_start(void) { }
-static inline void cpu_load_update_nohz_stop(void) { }
-#endif
-
-extern void dump_cpu_task(int cpu);
-
+struct nsproxy;
+struct perf_event_context;
+struct pid_namespace;
+struct pipe_inode_info;
+struct rcu_node;
+struct reclaim_state;
+struct robust_list_head;
+struct sched_attr;
+struct sched_param;
struct seq_file;
-struct cfs_rq;
+struct sighand_struct;
+struct signal_struct;
+struct task_delay_info;
struct task_group;
-#ifdef CONFIG_SCHED_DEBUG
-extern void proc_sched_show_task(struct task_struct *p, struct seq_file *m);
-extern void proc_sched_set_task(struct task_struct *p);
-#endif
/*
* Task state bitmask. NOTE! These bits are also
* modifying one set can't modify the other one by
* mistake.
*/
-#define TASK_RUNNING 0
-#define TASK_INTERRUPTIBLE 1
-#define TASK_UNINTERRUPTIBLE 2
-#define __TASK_STOPPED 4
-#define __TASK_TRACED 8
-/* in tsk->exit_state */
-#define EXIT_DEAD 16
-#define EXIT_ZOMBIE 32
-#define EXIT_TRACE (EXIT_ZOMBIE | EXIT_DEAD)
-/* in tsk->state again */
-#define TASK_DEAD 64
-#define TASK_WAKEKILL 128
-#define TASK_WAKING 256
-#define TASK_PARKED 512
-#define TASK_NOLOAD 1024
-#define TASK_NEW 2048
-#define TASK_STATE_MAX 4096
-
-#define TASK_STATE_TO_CHAR_STR "RSDTtXZxKWPNn"
-
-extern char ___assert_task_state[1 - 2*!!(
- sizeof(TASK_STATE_TO_CHAR_STR)-1 != ilog2(TASK_STATE_MAX)+1)];
-
-/* Convenience macros for the sake of set_current_state */
-#define TASK_KILLABLE (TASK_WAKEKILL | TASK_UNINTERRUPTIBLE)
-#define TASK_STOPPED (TASK_WAKEKILL | __TASK_STOPPED)
-#define TASK_TRACED (TASK_WAKEKILL | __TASK_TRACED)
-
-#define TASK_IDLE (TASK_UNINTERRUPTIBLE | TASK_NOLOAD)
-
-/* Convenience macros for the sake of wake_up */
-#define TASK_NORMAL (TASK_INTERRUPTIBLE | TASK_UNINTERRUPTIBLE)
-#define TASK_ALL (TASK_NORMAL | __TASK_STOPPED | __TASK_TRACED)
-
-/* get_task_state() */
-#define TASK_REPORT (TASK_RUNNING | TASK_INTERRUPTIBLE | \
- TASK_UNINTERRUPTIBLE | __TASK_STOPPED | \
- __TASK_TRACED | EXIT_ZOMBIE | EXIT_DEAD)
-
-#define task_is_traced(task) ((task->state & __TASK_TRACED) != 0)
-#define task_is_stopped(task) ((task->state & __TASK_STOPPED) != 0)
-#define task_is_stopped_or_traced(task) \
- ((task->state & (__TASK_STOPPED | __TASK_TRACED)) != 0)
-#define task_contributes_to_load(task) \
- ((task->state & TASK_UNINTERRUPTIBLE) != 0 && \
- (task->flags & PF_FROZEN) == 0 && \
- (task->state & TASK_NOLOAD) == 0)
+
+/* Used in tsk->state: */
+#define TASK_RUNNING 0
+#define TASK_INTERRUPTIBLE 1
+#define TASK_UNINTERRUPTIBLE 2
+#define __TASK_STOPPED 4
+#define __TASK_TRACED 8
+/* Used in tsk->exit_state: */
+#define EXIT_DEAD 16
+#define EXIT_ZOMBIE 32
+#define EXIT_TRACE (EXIT_ZOMBIE | EXIT_DEAD)
+/* Used in tsk->state again: */
+#define TASK_DEAD 64
+#define TASK_WAKEKILL 128
+#define TASK_WAKING 256
+#define TASK_PARKED 512
+#define TASK_NOLOAD 1024
+#define TASK_NEW 2048
+#define TASK_STATE_MAX 4096
+
+#define TASK_STATE_TO_CHAR_STR "RSDTtXZxKWPNn"
+
+/* Convenience macros for the sake of set_current_state: */
+#define TASK_KILLABLE (TASK_WAKEKILL | TASK_UNINTERRUPTIBLE)
+#define TASK_STOPPED (TASK_WAKEKILL | __TASK_STOPPED)
+#define TASK_TRACED (TASK_WAKEKILL | __TASK_TRACED)
+
+#define TASK_IDLE (TASK_UNINTERRUPTIBLE | TASK_NOLOAD)
+
+/* Convenience macros for the sake of wake_up(): */
+#define TASK_NORMAL (TASK_INTERRUPTIBLE | TASK_UNINTERRUPTIBLE)
+#define TASK_ALL (TASK_NORMAL | __TASK_STOPPED | __TASK_TRACED)
+
+/* get_task_state(): */
+#define TASK_REPORT (TASK_RUNNING | TASK_INTERRUPTIBLE | \
+ TASK_UNINTERRUPTIBLE | __TASK_STOPPED | \
+ __TASK_TRACED | EXIT_ZOMBIE | EXIT_DEAD)
+
+#define task_is_traced(task) ((task->state & __TASK_TRACED) != 0)
+
+#define task_is_stopped(task) ((task->state & __TASK_STOPPED) != 0)
+
+#define task_is_stopped_or_traced(task) ((task->state & (__TASK_STOPPED | __TASK_TRACED)) != 0)
+
+#define task_contributes_to_load(task) ((task->state & TASK_UNINTERRUPTIBLE) != 0 && \
+ (task->flags & PF_FROZEN) == 0 && \
+ (task->state & TASK_NOLOAD) == 0)
#ifdef CONFIG_DEBUG_ATOMIC_SLEEP
*
* Also see the comments of try_to_wake_up().
*/
-#define __set_current_state(state_value) \
- do { current->state = (state_value); } while (0)
-#define set_current_state(state_value) \
- smp_store_mb(current->state, (state_value))
-
-#endif
-
-/* Task command name length */
-#define TASK_COMM_LEN 16
-
-#include <linux/spinlock.h>
-
-/*
- * This serializes "schedule()" and also protects
- * the run-queue from deletions/modifications (but
- * _adding_ to the beginning of the run-queue has
- * a separate lock).
- */
-extern rwlock_t tasklist_lock;
-extern spinlock_t mmlist_lock;
-
-struct task_struct;
-
-#ifdef CONFIG_PROVE_RCU
-extern int lockdep_tasklist_lock_is_held(void);
-#endif /* #ifdef CONFIG_PROVE_RCU */
-
-extern void sched_init(void);
-extern void sched_init_smp(void);
-extern asmlinkage void schedule_tail(struct task_struct *prev);
-extern void init_idle(struct task_struct *idle, int cpu);
-extern void init_idle_bootup_task(struct task_struct *idle);
-
-extern cpumask_var_t cpu_isolated_map;
-
-extern int runqueue_is_locked(int cpu);
-
-#if defined(CONFIG_SMP) && defined(CONFIG_NO_HZ_COMMON)
-extern void nohz_balance_enter_idle(int cpu);
-extern void set_cpu_sd_state_idle(void);
-extern int get_nohz_timer_target(void);
-#else
-static inline void nohz_balance_enter_idle(int cpu) { }
-static inline void set_cpu_sd_state_idle(void) { }
+#define __set_current_state(state_value) do { current->state = (state_value); } while (0)
+#define set_current_state(state_value) smp_store_mb(current->state, (state_value))
#endif
-/*
- * Only dump TASK_* tasks. (0 for all tasks)
- */
-extern void show_state_filter(unsigned long state_filter);
-
-static inline void show_state(void)
-{
- show_state_filter(0);
-}
+/* Task command name length: */
+#define TASK_COMM_LEN 16
-extern void show_regs(struct pt_regs *);
-
-/*
- * TASK is a pointer to the task whose backtrace we want to see (or NULL for current
- * task), SP is the stack pointer of the first frame that should be shown in the back
- * trace (or NULL if the entire call-chain of the task should be shown).
- */
-extern void show_stack(struct task_struct *task, unsigned long *sp);
+extern cpumask_var_t cpu_isolated_map;
-extern void cpu_init (void);
-extern void trap_init(void);
-extern void update_process_times(int user);
extern void scheduler_tick(void);
-extern int sched_cpu_starting(unsigned int cpu);
-extern int sched_cpu_activate(unsigned int cpu);
-extern int sched_cpu_deactivate(unsigned int cpu);
-
-#ifdef CONFIG_HOTPLUG_CPU
-extern int sched_cpu_dying(unsigned int cpu);
-#else
-# define sched_cpu_dying NULL
-#endif
-
-extern void sched_show_task(struct task_struct *p);
-
-#ifdef CONFIG_LOCKUP_DETECTOR
-extern void touch_softlockup_watchdog_sched(void);
-extern void touch_softlockup_watchdog(void);
-extern void touch_softlockup_watchdog_sync(void);
-extern void touch_all_softlockup_watchdogs(void);
-extern int proc_dowatchdog_thresh(struct ctl_table *table, int write,
- void __user *buffer,
- size_t *lenp, loff_t *ppos);
-extern unsigned int softlockup_panic;
-extern unsigned int hardlockup_panic;
-void lockup_detector_init(void);
-#else
-static inline void touch_softlockup_watchdog_sched(void)
-{
-}
-static inline void touch_softlockup_watchdog(void)
-{
-}
-static inline void touch_softlockup_watchdog_sync(void)
-{
-}
-static inline void touch_all_softlockup_watchdogs(void)
-{
-}
-static inline void lockup_detector_init(void)
-{
-}
-#endif
-
-#ifdef CONFIG_DETECT_HUNG_TASK
-void reset_hung_task_detector(void);
-#else
-static inline void reset_hung_task_detector(void)
-{
-}
-#endif
-
-/* Attach to any functions which should be ignored in wchan output. */
-#define __sched __attribute__((__section__(".sched.text")))
-/* Linker adds these: start and end of __sched functions */
-extern char __sched_text_start[], __sched_text_end[];
+#define MAX_SCHEDULE_TIMEOUT LONG_MAX
-/* Is this address in the __sched functions? */
-extern int in_sched_functions(unsigned long addr);
-
-#define MAX_SCHEDULE_TIMEOUT LONG_MAX
-extern signed long schedule_timeout(signed long timeout);
-extern signed long schedule_timeout_interruptible(signed long timeout);
-extern signed long schedule_timeout_killable(signed long timeout);
-extern signed long schedule_timeout_uninterruptible(signed long timeout);
-extern signed long schedule_timeout_idle(signed long timeout);
+extern long schedule_timeout(long timeout);
+extern long schedule_timeout_interruptible(long timeout);
+extern long schedule_timeout_killable(long timeout);
+extern long schedule_timeout_uninterruptible(long timeout);
+extern long schedule_timeout_idle(long timeout);
asmlinkage void schedule(void);
extern void schedule_preempt_disabled(void);
extern long io_schedule_timeout(long timeout);
extern void io_schedule(void);
-void __noreturn do_task_dead(void);
-
-struct nsproxy;
-struct user_namespace;
-
-#ifdef CONFIG_MMU
-extern void arch_pick_mmap_layout(struct mm_struct *mm);
-extern unsigned long
-arch_get_unmapped_area(struct file *, unsigned long, unsigned long,
- unsigned long, unsigned long);
-extern unsigned long
-arch_get_unmapped_area_topdown(struct file *filp, unsigned long addr,
- unsigned long len, unsigned long pgoff,
- unsigned long flags);
-#else
-static inline void arch_pick_mmap_layout(struct mm_struct *mm) {}
-#endif
-
-#define SUID_DUMP_DISABLE 0 /* No setuid dumping */
-#define SUID_DUMP_USER 1 /* Dump as user of process */
-#define SUID_DUMP_ROOT 2 /* Dump as root */
-
-/* mm flags */
-
-/* for SUID_DUMP_* above */
-#define MMF_DUMPABLE_BITS 2
-#define MMF_DUMPABLE_MASK ((1 << MMF_DUMPABLE_BITS) - 1)
-
-extern void set_dumpable(struct mm_struct *mm, int value);
-/*
- * This returns the actual value of the suid_dumpable flag. For things
- * that are using this for checking for privilege transitions, it must
- * test against SUID_DUMP_USER rather than treating it as a boolean
- * value.
- */
-static inline int __get_dumpable(unsigned long mm_flags)
-{
- return mm_flags & MMF_DUMPABLE_MASK;
-}
-
-static inline int get_dumpable(struct mm_struct *mm)
-{
- return __get_dumpable(mm->flags);
-}
-
-/* coredump filter bits */
-#define MMF_DUMP_ANON_PRIVATE 2
-#define MMF_DUMP_ANON_SHARED 3
-#define MMF_DUMP_MAPPED_PRIVATE 4
-#define MMF_DUMP_MAPPED_SHARED 5
-#define MMF_DUMP_ELF_HEADERS 6
-#define MMF_DUMP_HUGETLB_PRIVATE 7
-#define MMF_DUMP_HUGETLB_SHARED 8
-#define MMF_DUMP_DAX_PRIVATE 9
-#define MMF_DUMP_DAX_SHARED 10
-
-#define MMF_DUMP_FILTER_SHIFT MMF_DUMPABLE_BITS
-#define MMF_DUMP_FILTER_BITS 9
-#define MMF_DUMP_FILTER_MASK \
- (((1 << MMF_DUMP_FILTER_BITS) - 1) << MMF_DUMP_FILTER_SHIFT)
-#define MMF_DUMP_FILTER_DEFAULT \
- ((1 << MMF_DUMP_ANON_PRIVATE) | (1 << MMF_DUMP_ANON_SHARED) |\
- (1 << MMF_DUMP_HUGETLB_PRIVATE) | MMF_DUMP_MASK_DEFAULT_ELF)
-
-#ifdef CONFIG_CORE_DUMP_DEFAULT_ELF_HEADERS
-# define MMF_DUMP_MASK_DEFAULT_ELF (1 << MMF_DUMP_ELF_HEADERS)
-#else
-# define MMF_DUMP_MASK_DEFAULT_ELF 0
-#endif
- /* leave room for more dump flags */
-#define MMF_VM_MERGEABLE 16 /* KSM may merge identical pages */
-#define MMF_VM_HUGEPAGE 17 /* set when VM_HUGEPAGE is set on vma */
-/*
- * This one-shot flag is dropped due to necessity of changing exe once again
- * on NFS restore
- */
-//#define MMF_EXE_FILE_CHANGED 18 /* see prctl_set_mm_exe_file() */
-
-#define MMF_HAS_UPROBES 19 /* has uprobes */
-#define MMF_RECALC_UPROBES 20 /* MMF_HAS_UPROBES can be wrong */
-#define MMF_OOM_SKIP 21 /* mm is of no interest for the OOM killer */
-#define MMF_UNSTABLE 22 /* mm is unstable for copy_from_user */
-#define MMF_HUGE_ZERO_PAGE 23 /* mm has ever used the global huge zero page */
-
-#define MMF_INIT_MASK (MMF_DUMPABLE_MASK | MMF_DUMP_FILTER_MASK)
-
-struct sighand_struct {
- atomic_t count;
- struct k_sigaction action[_NSIG];
- spinlock_t siglock;
- wait_queue_head_t signalfd_wqh;
-};
-
-struct pacct_struct {
- int ac_flag;
- long ac_exitcode;
- unsigned long ac_mem;
- u64 ac_utime, ac_stime;
- unsigned long ac_minflt, ac_majflt;
-};
-
-struct cpu_itimer {
- u64 expires;
- u64 incr;
-};
-
/**
* struct prev_cputime - snaphsot of system and user cputime
* @utime: time spent in user mode
*/
struct prev_cputime {
#ifndef CONFIG_VIRT_CPU_ACCOUNTING_NATIVE
- u64 utime;
- u64 stime;
- raw_spinlock_t lock;
+ u64 utime;
+ u64 stime;
+ raw_spinlock_t lock;
#endif
};
-static inline void prev_cputime_init(struct prev_cputime *prev)
-{
-#ifndef CONFIG_VIRT_CPU_ACCOUNTING_NATIVE
- prev->utime = prev->stime = 0;
- raw_spin_lock_init(&prev->lock);
-#endif
-}
-
/**
* struct task_cputime - collected CPU time counts
* @utime: time spent in user mode, in nanoseconds
* these counts together and treat all three of them in parallel.
*/
struct task_cputime {
- u64 utime;
- u64 stime;
- unsigned long long sum_exec_runtime;
+ u64 utime;
+ u64 stime;
+ unsigned long long sum_exec_runtime;
};
-/* Alternate field names when used to cache expirations. */
-#define virt_exp utime
-#define prof_exp stime
-#define sched_exp sum_exec_runtime
+/* Alternate field names when used on cache expirations: */
+#define virt_exp utime
+#define prof_exp stime
+#define sched_exp sum_exec_runtime
-/*
- * This is the atomic variant of task_cputime, which can be used for
- * storing and updating task_cputime statistics without locking.
- */
-struct task_cputime_atomic {
- atomic64_t utime;
- atomic64_t stime;
- atomic64_t sum_exec_runtime;
-};
+struct sched_info {
+#ifdef CONFIG_SCHED_INFO
+ /* Cumulative counters: */
+
+ /* # of times we have run on this CPU: */
+ unsigned long pcount;
+
+ /* Time spent waiting on a runqueue: */
+ unsigned long long run_delay;
+
+ /* Timestamps: */
+
+ /* When did we last run on a CPU? */
+ unsigned long long last_arrival;
-#define INIT_CPUTIME_ATOMIC \
- (struct task_cputime_atomic) { \
- .utime = ATOMIC64_INIT(0), \
- .stime = ATOMIC64_INIT(0), \
- .sum_exec_runtime = ATOMIC64_INIT(0), \
- }
+ /* When were we last queued to run? */
+ unsigned long long last_queued;
-#define PREEMPT_DISABLED (PREEMPT_DISABLE_OFFSET + PREEMPT_ENABLED)
+#endif /* CONFIG_SCHED_INFO */
+};
/*
- * Disable preemption until the scheduler is running -- use an unconditional
- * value so that it also works on !PREEMPT_COUNT kernels.
+ * Integer metrics need fixed point arithmetic, e.g., sched/fair
+ * has a few: load, load_avg, util_avg, freq, and capacity.
*
- * Reset by start_kernel()->sched_init()->init_idle()->init_idle_preempt_count().
+ * We define a basic fixed point arithmetic range, and then formalize
+ * all these metrics based on that basic range.
*/
-#define INIT_PREEMPT_COUNT PREEMPT_OFFSET
+# define SCHED_FIXEDPOINT_SHIFT 10
+# define SCHED_FIXEDPOINT_SCALE (1L << SCHED_FIXEDPOINT_SHIFT)
+
+struct load_weight {
+ unsigned long weight;
+ u32 inv_weight;
+};
/*
- * Initial preempt_count value; reflects the preempt_count schedule invariant
- * which states that during context switches:
+ * The load_avg/util_avg accumulates an infinite geometric series
+ * (see __update_load_avg() in kernel/sched/fair.c).
*
- * preempt_count() == 2*PREEMPT_DISABLE_OFFSET
+ * [load_avg definition]
*
- * Note: PREEMPT_DISABLE_OFFSET is 0 for !PREEMPT_COUNT kernels.
- * Note: See finish_task_switch().
- */
-#define FORK_PREEMPT_COUNT (2*PREEMPT_DISABLE_OFFSET + PREEMPT_ENABLED)
-
-/**
- * struct thread_group_cputimer - thread group interval timer counts
- * @cputime_atomic: atomic thread group interval timers.
- * @running: true when there are timers running and
- * @cputime_atomic receives updates.
- * @checking_timer: true when a thread in the group is in the
- * process of checking for thread group timers.
+ * load_avg = runnable% * scale_load_down(load)
+ *
+ * where runnable% is the time ratio that a sched_entity is runnable.
+ * For cfs_rq, it is the aggregated load_avg of all runnable and
+ * blocked sched_entities.
+ *
+ * load_avg may also take frequency scaling into account:
+ *
+ * load_avg = runnable% * scale_load_down(load) * freq%
*
- * This structure contains the version of task_cputime, above, that is
- * used for thread group CPU timer calculations.
+ * where freq% is the CPU frequency normalized to the highest frequency.
+ *
+ * [util_avg definition]
+ *
+ * util_avg = running% * SCHED_CAPACITY_SCALE
+ *
+ * where running% is the time ratio that a sched_entity is running on
+ * a CPU. For cfs_rq, it is the aggregated util_avg of all runnable
+ * and blocked sched_entities.
+ *
+ * util_avg may also factor frequency scaling and CPU capacity scaling:
+ *
+ * util_avg = running% * SCHED_CAPACITY_SCALE * freq% * capacity%
+ *
+ * where freq% is the same as above, and capacity% is the CPU capacity
+ * normalized to the greatest capacity (due to uarch differences, etc).
+ *
+ * N.B., the above ratios (runnable%, running%, freq%, and capacity%)
+ * themselves are in the range of [0, 1]. To do fixed point arithmetics,
+ * we therefore scale them to as large a range as necessary. This is for
+ * example reflected by util_avg's SCHED_CAPACITY_SCALE.
+ *
+ * [Overflow issue]
+ *
+ * The 64-bit load_sum can have 4353082796 (=2^64/47742/88761) entities
+ * with the highest load (=88761), always runnable on a single cfs_rq,
+ * and should not overflow as the number already hits PID_MAX_LIMIT.
+ *
+ * For all other cases (including 32-bit kernels), struct load_weight's
+ * weight will overflow first before we do, because:
+ *
+ * Max(load_avg) <= Max(load.weight)
+ *
+ * Then it is the load_weight's responsibility to consider overflow
+ * issues.
*/
-struct thread_group_cputimer {
- struct task_cputime_atomic cputime_atomic;
- bool running;
- bool checking_timer;
+struct sched_avg {
+ u64 last_update_time;
+ u64 load_sum;
+ u32 util_sum;
+ u32 period_contrib;
+ unsigned long load_avg;
+ unsigned long util_avg;
};
-#include <linux/rwsem.h>
-struct autogroup;
-
-/*
- * NOTE! "signal_struct" does not have its own
- * locking, because a shared signal_struct always
- * implies a shared sighand_struct, so locking
- * sighand_struct is always a proper superset of
- * the locking of signal_struct.
- */
-struct signal_struct {
- atomic_t sigcnt;
- atomic_t live;
- int nr_threads;
- struct list_head thread_head;
-
- wait_queue_head_t wait_chldexit; /* for wait4() */
-
- /* current thread group signal load-balancing target: */
- struct task_struct *curr_target;
-
- /* shared signal handling: */
- struct sigpending shared_pending;
-
- /* thread group exit support */
- int group_exit_code;
- /* overloaded:
- * - notify group_exit_task when ->count is equal to notify_count
- * - everyone except group_exit_task is stopped during signal delivery
- * of fatal signals, group_exit_task processes the signal.
- */
- int notify_count;
- struct task_struct *group_exit_task;
-
- /* thread group stop support, overloads group_exit_code too */
- int group_stop_count;
- unsigned int flags; /* see SIGNAL_* flags below */
+struct sched_statistics {
+#ifdef CONFIG_SCHEDSTATS
+ u64 wait_start;
+ u64 wait_max;
+ u64 wait_count;
+ u64 wait_sum;
+ u64 iowait_count;
+ u64 iowait_sum;
+
+ u64 sleep_start;
+ u64 sleep_max;
+ s64 sum_sleep_runtime;
+
+ u64 block_start;
+ u64 block_max;
+ u64 exec_max;
+ u64 slice_max;
+
+ u64 nr_migrations_cold;
+ u64 nr_failed_migrations_affine;
+ u64 nr_failed_migrations_running;
+ u64 nr_failed_migrations_hot;
+ u64 nr_forced_migrations;
+
+ u64 nr_wakeups;
+ u64 nr_wakeups_sync;
+ u64 nr_wakeups_migrate;
+ u64 nr_wakeups_local;
+ u64 nr_wakeups_remote;
+ u64 nr_wakeups_affine;
+ u64 nr_wakeups_affine_attempts;
+ u64 nr_wakeups_passive;
+ u64 nr_wakeups_idle;
+#endif
+};
- /*
- * PR_SET_CHILD_SUBREAPER marks a process, like a service
- * manager, to re-parent orphan (double-forking) child processes
- * to this process instead of 'init'. The service manager is
- * able to receive SIGCHLD signals and is able to investigate
- * the process until it calls wait(). All children of this
- * process will inherit a flag if they should look for a
- * child_subreaper process at exit.
- */
- unsigned int is_child_subreaper:1;
- unsigned int has_child_subreaper:1;
+struct sched_entity {
+ /* For load-balancing: */
+ struct load_weight load;
+ struct rb_node run_node;
+ struct list_head group_node;
+ unsigned int on_rq;
-#ifdef CONFIG_POSIX_TIMERS
+ u64 exec_start;
+ u64 sum_exec_runtime;
+ u64 vruntime;
+ u64 prev_sum_exec_runtime;
- /* POSIX.1b Interval Timers */
- int posix_timer_id;
- struct list_head posix_timers;
+ u64 nr_migrations;
- /* ITIMER_REAL timer for the process */
- struct hrtimer real_timer;
- ktime_t it_real_incr;
+ struct sched_statistics statistics;
- /*
- * ITIMER_PROF and ITIMER_VIRTUAL timers for the process, we use
- * CPUCLOCK_PROF and CPUCLOCK_VIRT for indexing array as these
- * values are defined to 0 and 1 respectively
- */
- struct cpu_itimer it[2];
+#ifdef CONFIG_FAIR_GROUP_SCHED
+ int depth;
+ struct sched_entity *parent;
+ /* rq on which this entity is (to be) queued: */
+ struct cfs_rq *cfs_rq;
+ /* rq "owned" by this entity/group: */
+ struct cfs_rq *my_q;
+#endif
+#ifdef CONFIG_SMP
/*
- * Thread group totals for process CPU timers.
- * See thread_group_cputimer(), et al, for details.
+ * Per entity load average tracking.
+ *
+ * Put into separate cache line so it does not
+ * collide with read-mostly values above.
*/
- struct thread_group_cputimer cputimer;
-
- /* Earliest-expiration cache. */
- struct task_cputime cputime_expires;
-
- struct list_head cpu_timers[3];
-
+ struct sched_avg avg ____cacheline_aligned_in_smp;
#endif
+};
- struct pid *leader_pid;
-
-#ifdef CONFIG_NO_HZ_FULL
- atomic_t tick_dep_mask;
+struct sched_rt_entity {
+ struct list_head run_list;
+ unsigned long timeout;
+ unsigned long watchdog_stamp;
+ unsigned int time_slice;
+ unsigned short on_rq;
+ unsigned short on_list;
+
+ struct sched_rt_entity *back;
+#ifdef CONFIG_RT_GROUP_SCHED
+ struct sched_rt_entity *parent;
+ /* rq on which this entity is (to be) queued: */
+ struct rt_rq *rt_rq;
+ /* rq "owned" by this entity/group: */
+ struct rt_rq *my_q;
#endif
+};
- struct pid *tty_old_pgrp;
-
- /* boolean value for session group leader */
- int leader;
-
- struct tty_struct *tty; /* NULL if no tty */
+struct sched_dl_entity {
+ struct rb_node rb_node;
-#ifdef CONFIG_SCHED_AUTOGROUP
- struct autogroup *autogroup;
-#endif
/*
- * Cumulative resource counters for dead threads in the group,
- * and for reaped dead child processes forked by this group.
- * Live threads maintain their own counters and add to these
- * in __exit_signal, except for the group leader.
+ * Original scheduling parameters. Copied here from sched_attr
+ * during sched_setattr(), they will remain the same until
+ * the next sched_setattr().
*/
- seqlock_t stats_lock;
- u64 utime, stime, cutime, cstime;
- u64 gtime;
- u64 cgtime;
- struct prev_cputime prev_cputime;
- unsigned long nvcsw, nivcsw, cnvcsw, cnivcsw;
- unsigned long min_flt, maj_flt, cmin_flt, cmaj_flt;
- unsigned long inblock, oublock, cinblock, coublock;
- unsigned long maxrss, cmaxrss;
- struct task_io_accounting ioac;
+ u64 dl_runtime; /* Maximum runtime for each instance */
+ u64 dl_deadline; /* Relative deadline of each instance */
+ u64 dl_period; /* Separation of two instances (period) */
+ u64 dl_bw; /* dl_runtime / dl_deadline */
/*
- * Cumulative ns of schedule CPU time fo dead threads in the
- * group, not including a zombie group leader, (This only differs
- * from jiffies_to_ns(utime + stime) if sched_clock uses something
- * other than jiffies.)
+ * Actual scheduling parameters. Initialized with the values above,
+ * they are continously updated during task execution. Note that
+ * the remaining runtime could be < 0 in case we are in overrun.
*/
- unsigned long long sum_sched_runtime;
+ s64 runtime; /* Remaining runtime for this instance */
+ u64 deadline; /* Absolute deadline for this instance */
+ unsigned int flags; /* Specifying the scheduler behaviour */
/*
- * We don't bother to synchronize most readers of this at all,
- * because there is no reader checking a limit that actually needs
- * to get both rlim_cur and rlim_max atomically, and either one
- * alone is a single word that can safely be read normally.
- * getrlimit/setrlimit use task_lock(current->group_leader) to
- * protect this instead of the siglock, because they really
- * have no need to disable irqs.
+ * Some bool flags:
+ *
+ * @dl_throttled tells if we exhausted the runtime. If so, the
+ * task has to wait for a replenishment to be performed at the
+ * next firing of dl_timer.
+ *
+ * @dl_boosted tells if we are boosted due to DI. If so we are
+ * outside bandwidth enforcement mechanism (but only until we
+ * exit the critical section);
+ *
+ * @dl_yielded tells if task gave up the CPU before consuming
+ * all its available runtime during the last job.
*/
- struct rlimit rlim[RLIM_NLIMITS];
-
-#ifdef CONFIG_BSD_PROCESS_ACCT
- struct pacct_struct pacct; /* per-process accounting information */
-#endif
-#ifdef CONFIG_TASKSTATS
- struct taskstats *stats;
-#endif
-#ifdef CONFIG_AUDIT
- unsigned audit_tty;
- struct tty_audit_buf *tty_audit_buf;
-#endif
+ int dl_throttled;
+ int dl_boosted;
+ int dl_yielded;
/*
- * Thread is the potential origin of an oom condition; kill first on
- * oom
+ * Bandwidth enforcement timer. Each -deadline task has its
+ * own bandwidth to be enforced, thus we need one timer per task.
*/
- bool oom_flag_origin;
- short oom_score_adj; /* OOM kill score adjustment */
- short oom_score_adj_min; /* OOM kill score adjustment min value.
- * Only settable by CAP_SYS_RESOURCE. */
- struct mm_struct *oom_mm; /* recorded mm when the thread group got
- * killed by the oom killer */
-
- struct mutex cred_guard_mutex; /* guard against foreign influences on
- * credential calculations
- * (notably. ptrace) */
+ struct hrtimer dl_timer;
};
-/*
- * Bits in flags field of signal_struct.
- */
-#define SIGNAL_STOP_STOPPED 0x00000001 /* job control stop in effect */
-#define SIGNAL_STOP_CONTINUED 0x00000002 /* SIGCONT since WCONTINUED reap */
-#define SIGNAL_GROUP_EXIT 0x00000004 /* group exit in progress */
-#define SIGNAL_GROUP_COREDUMP 0x00000008 /* coredump in progress */
-/*
- * Pending notifications to parent.
- */
-#define SIGNAL_CLD_STOPPED 0x00000010
-#define SIGNAL_CLD_CONTINUED 0x00000020
-#define SIGNAL_CLD_MASK (SIGNAL_CLD_STOPPED|SIGNAL_CLD_CONTINUED)
-
-#define SIGNAL_UNKILLABLE 0x00000040 /* for init: ignore fatal signals */
+union rcu_special {
+ struct {
+ u8 blocked;
+ u8 need_qs;
+ u8 exp_need_qs;
-#define SIGNAL_STOP_MASK (SIGNAL_CLD_MASK | SIGNAL_STOP_STOPPED | \
- SIGNAL_STOP_CONTINUED)
+ /* Otherwise the compiler can store garbage here: */
+ u8 pad;
+ } b; /* Bits. */
+ u32 s; /* Set of bits. */
+};
-static inline void signal_set_stop_flags(struct signal_struct *sig,
- unsigned int flags)
-{
- WARN_ON(sig->flags & (SIGNAL_GROUP_EXIT|SIGNAL_GROUP_COREDUMP));
- sig->flags = (sig->flags & ~SIGNAL_STOP_MASK) | flags;
-}
+enum perf_event_task_context {
+ perf_invalid_context = -1,
+ perf_hw_context = 0,
+ perf_sw_context,
+ perf_nr_task_contexts,
+};
-/* If true, all threads except ->group_exit_task have pending SIGKILL */
-static inline int signal_group_exit(const struct signal_struct *sig)
-{
- return (sig->flags & SIGNAL_GROUP_EXIT) ||
- (sig->group_exit_task != NULL);
-}
+struct wake_q_node {
+ struct wake_q_node *next;
+};
-/*
- * Some day this will be a full-fledged user tracking system..
- */
-struct user_struct {
- atomic_t __count; /* reference count */
- atomic_t processes; /* How many processes does this user have? */
- atomic_t sigpending; /* How many pending signals does this user have? */
-#ifdef CONFIG_FANOTIFY
- atomic_t fanotify_listeners;
+struct task_struct {
+#ifdef CONFIG_THREAD_INFO_IN_TASK
+ /*
+ * For reasons of header soup (see current_thread_info()), this
+ * must be the first element of task_struct.
+ */
+ struct thread_info thread_info;
#endif
-#ifdef CONFIG_EPOLL
- atomic_long_t epoll_watches; /* The number of file descriptors currently watched */
+ /* -1 unrunnable, 0 runnable, >0 stopped: */
+ volatile long state;
+ void *stack;
+ atomic_t usage;
+ /* Per task flags (PF_*), defined further below: */
+ unsigned int flags;
+ unsigned int ptrace;
+
+#ifdef CONFIG_SMP
+ struct llist_node wake_entry;
+ int on_cpu;
+#ifdef CONFIG_THREAD_INFO_IN_TASK
+ /* Current CPU: */
+ unsigned int cpu;
#endif
-#ifdef CONFIG_POSIX_MQUEUE
- /* protected by mq_lock */
- unsigned long mq_bytes; /* How many bytes can be allocated to mqueue? */
+ unsigned int wakee_flips;
+ unsigned long wakee_flip_decay_ts;
+ struct task_struct *last_wakee;
+
+ int wake_cpu;
#endif
- unsigned long locked_shm; /* How many pages of mlocked shm ? */
- unsigned long unix_inflight; /* How many files in flight in unix sockets */
- atomic_long_t pipe_bufs; /* how many pages are allocated in pipe buffers */
+ int on_rq;
+
+ int prio;
+ int static_prio;
+ int normal_prio;
+ unsigned int rt_priority;
-#ifdef CONFIG_KEYS
- struct key *uid_keyring; /* UID specific keyring */
- struct key *session_keyring; /* UID's default session keyring */
+ const struct sched_class *sched_class;
+ struct sched_entity se;
+ struct sched_rt_entity rt;
+#ifdef CONFIG_CGROUP_SCHED
+ struct task_group *sched_task_group;
#endif
+ struct sched_dl_entity dl;
- /* Hash table maintenance information */
- struct hlist_node uidhash_node;
- kuid_t uid;
+#ifdef CONFIG_PREEMPT_NOTIFIERS
+ /* List of struct preempt_notifier: */
+ struct hlist_head preempt_notifiers;
+#endif
-#if defined(CONFIG_PERF_EVENTS) || defined(CONFIG_BPF_SYSCALL)
- atomic_long_t locked_vm;
+#ifdef CONFIG_BLK_DEV_IO_TRACE
+ unsigned int btrace_seq;
#endif
-};
-extern int uids_sysfs_init(void);
+ unsigned int policy;
+ int nr_cpus_allowed;
+ cpumask_t cpus_allowed;
-extern struct user_struct *find_user(kuid_t);
+#ifdef CONFIG_PREEMPT_RCU
+ int rcu_read_lock_nesting;
+ union rcu_special rcu_read_unlock_special;
+ struct list_head rcu_node_entry;
+ struct rcu_node *rcu_blocked_node;
+#endif /* #ifdef CONFIG_PREEMPT_RCU */
-extern struct user_struct root_user;
-#define INIT_USER (&root_user)
+#ifdef CONFIG_TASKS_RCU
+ unsigned long rcu_tasks_nvcsw;
+ bool rcu_tasks_holdout;
+ struct list_head rcu_tasks_holdout_list;
+ int rcu_tasks_idle_cpu;
+#endif /* #ifdef CONFIG_TASKS_RCU */
+ struct sched_info sched_info;
-struct backing_dev_info;
-struct reclaim_state;
+ struct list_head tasks;
+#ifdef CONFIG_SMP
+ struct plist_node pushable_tasks;
+ struct rb_node pushable_dl_tasks;
+#endif
-#ifdef CONFIG_SCHED_INFO
-struct sched_info {
- /* cumulative counters */
- unsigned long pcount; /* # of times run on this cpu */
- unsigned long long run_delay; /* time spent waiting on a runqueue */
+ struct mm_struct *mm;
+ struct mm_struct *active_mm;
- /* timestamps */
- unsigned long long last_arrival,/* when we last ran on a cpu */
- last_queued; /* when we were last queued to run */
-};
-#endif /* CONFIG_SCHED_INFO */
+ /* Per-thread vma caching: */
+ struct vmacache vmacache;
-#ifdef CONFIG_TASK_DELAY_ACCT
-struct task_delay_info {
- spinlock_t lock;
- unsigned int flags; /* Private per-task flags */
+#ifdef SPLIT_RSS_COUNTING
+ struct task_rss_stat rss_stat;
+#endif
+ int exit_state;
+ int exit_code;
+ int exit_signal;
+ /* The signal sent when the parent dies: */
+ int pdeath_signal;
+ /* JOBCTL_*, siglock protected: */
+ unsigned long jobctl;
- /* For each stat XXX, add following, aligned appropriately
- *
- * struct timespec XXX_start, XXX_end;
- * u64 XXX_delay;
- * u32 XXX_count;
- *
- * Atomicity of updates to XXX_delay, XXX_count protected by
- * single lock above (split into XXX_lock if contention is an issue).
- */
+ /* Used for emulating ABI behavior of previous Linux versions: */
+ unsigned int personality;
- /*
- * XXX_count is incremented on every XXX operation, the delay
- * associated with the operation is added to XXX_delay.
- * XXX_delay contains the accumulated delay time in nanoseconds.
- */
- u64 blkio_start; /* Shared by blkio, swapin */
- u64 blkio_delay; /* wait for sync block io completion */
- u64 swapin_delay; /* wait for swapin block io completion */
- u32 blkio_count; /* total count of the number of sync block */
- /* io operations performed */
- u32 swapin_count; /* total count of the number of swapin block */
- /* io operations performed */
-
- u64 freepages_start;
- u64 freepages_delay; /* wait for memory reclaim */
- u32 freepages_count; /* total count of memory reclaim */
-};
-#endif /* CONFIG_TASK_DELAY_ACCT */
+ /* Scheduler bits, serialized by scheduler locks: */
+ unsigned sched_reset_on_fork:1;
+ unsigned sched_contributes_to_load:1;
+ unsigned sched_migrated:1;
+ unsigned sched_remote_wakeup:1;
+ /* Force alignment to the next boundary: */
+ unsigned :0;
-static inline int sched_info_on(void)
-{
-#ifdef CONFIG_SCHEDSTATS
- return 1;
-#elif defined(CONFIG_TASK_DELAY_ACCT)
- extern int delayacct_on;
- return delayacct_on;
-#else
- return 0;
-#endif
-}
+ /* Unserialized, strictly 'current' */
-#ifdef CONFIG_SCHEDSTATS
-void force_schedstat_enabled(void);
+ /* Bit to tell LSMs we're in execve(): */
+ unsigned in_execve:1;
+ unsigned in_iowait:1;
+#ifndef TIF_RESTORE_SIGMASK
+ unsigned restore_sigmask:1;
+#endif
+#ifdef CONFIG_MEMCG
+ unsigned memcg_may_oom:1;
+#ifndef CONFIG_SLOB
+ unsigned memcg_kmem_skip_account:1;
+#endif
+#endif
+#ifdef CONFIG_COMPAT_BRK
+ unsigned brk_randomized:1;
#endif
-enum cpu_idle_type {
- CPU_IDLE,
- CPU_NOT_IDLE,
- CPU_NEWLY_IDLE,
- CPU_MAX_IDLE_TYPES
-};
+ unsigned long atomic_flags; /* Flags requiring atomic access. */
-/*
- * Integer metrics need fixed point arithmetic, e.g., sched/fair
- * has a few: load, load_avg, util_avg, freq, and capacity.
- *
- * We define a basic fixed point arithmetic range, and then formalize
- * all these metrics based on that basic range.
- */
-# define SCHED_FIXEDPOINT_SHIFT 10
-# define SCHED_FIXEDPOINT_SCALE (1L << SCHED_FIXEDPOINT_SHIFT)
+ struct restart_block restart_block;
-/*
- * Increase resolution of cpu_capacity calculations
- */
-#define SCHED_CAPACITY_SHIFT SCHED_FIXEDPOINT_SHIFT
-#define SCHED_CAPACITY_SCALE (1L << SCHED_CAPACITY_SHIFT)
+ pid_t pid;
+ pid_t tgid;
-/*
- * Wake-queues are lists of tasks with a pending wakeup, whose
- * callers have already marked the task as woken internally,
- * and can thus carry on. A common use case is being able to
- * do the wakeups once the corresponding user lock as been
- * released.
- *
- * We hold reference to each task in the list across the wakeup,
- * thus guaranteeing that the memory is still valid by the time
- * the actual wakeups are performed in wake_up_q().
- *
- * One per task suffices, because there's never a need for a task to be
- * in two wake queues simultaneously; it is forbidden to abandon a task
- * in a wake queue (a call to wake_up_q() _must_ follow), so if a task is
- * already in a wake queue, the wakeup will happen soon and the second
- * waker can just skip it.
- *
- * The DEFINE_WAKE_Q macro declares and initializes the list head.
- * wake_up_q() does NOT reinitialize the list; it's expected to be
- * called near the end of a function. Otherwise, the list can be
- * re-initialized for later re-use by wake_q_init().
- *
- * Note that this can cause spurious wakeups. schedule() callers
- * must ensure the call is done inside a loop, confirming that the
- * wakeup condition has in fact occurred.
- */
-struct wake_q_node {
- struct wake_q_node *next;
-};
+#ifdef CONFIG_CC_STACKPROTECTOR
+ /* Canary value for the -fstack-protector GCC feature: */
+ unsigned long stack_canary;
+#endif
+ /*
+ * Pointers to the (original) parent process, youngest child, younger sibling,
+ * older sibling, respectively. (p->father can be replaced with
+ * p->real_parent->pid)
+ */
-struct wake_q_head {
- struct wake_q_node *first;
- struct wake_q_node **lastp;
-};
+ /* Real parent process: */
+ struct task_struct __rcu *real_parent;
+
+ /* Recipient of SIGCHLD, wait4() reports: */
+ struct task_struct __rcu *parent;
-#define WAKE_Q_TAIL ((struct wake_q_node *) 0x01)
+ /*
+ * Children/sibling form the list of natural children:
+ */
+ struct list_head children;
+ struct list_head sibling;
+ struct task_struct *group_leader;
-#define DEFINE_WAKE_Q(name) \
- struct wake_q_head name = { WAKE_Q_TAIL, &name.first }
+ /*
+ * 'ptraced' is the list of tasks this task is using ptrace() on.
+ *
+ * This includes both natural children and PTRACE_ATTACH targets.
+ * 'ptrace_entry' is this task's link on the p->parent->ptraced list.
+ */
+ struct list_head ptraced;
+ struct list_head ptrace_entry;
-static inline void wake_q_init(struct wake_q_head *head)
-{
- head->first = WAKE_Q_TAIL;
- head->lastp = &head->first;
-}
+ /* PID/PID hash table linkage. */
+ struct pid_link pids[PIDTYPE_MAX];
+ struct list_head thread_group;
+ struct list_head thread_node;
-extern void wake_q_add(struct wake_q_head *head,
- struct task_struct *task);
-extern void wake_up_q(struct wake_q_head *head);
+ struct completion *vfork_done;
-/*
- * sched-domains (multiprocessor balancing) declarations:
- */
-#ifdef CONFIG_SMP
-#define SD_LOAD_BALANCE 0x0001 /* Do load balancing on this domain. */
-#define SD_BALANCE_NEWIDLE 0x0002 /* Balance when about to become idle */
-#define SD_BALANCE_EXEC 0x0004 /* Balance on exec */
-#define SD_BALANCE_FORK 0x0008 /* Balance on fork, clone */
-#define SD_BALANCE_WAKE 0x0010 /* Balance on wakeup */
-#define SD_WAKE_AFFINE 0x0020 /* Wake task to waking CPU */
-#define SD_ASYM_CPUCAPACITY 0x0040 /* Groups have different max cpu capacities */
-#define SD_SHARE_CPUCAPACITY 0x0080 /* Domain members share cpu capacity */
-#define SD_SHARE_POWERDOMAIN 0x0100 /* Domain members share power domain */
-#define SD_SHARE_PKG_RESOURCES 0x0200 /* Domain members share cpu pkg resources */
-#define SD_SERIALIZE 0x0400 /* Only a single load balancing instance */
-#define SD_ASYM_PACKING 0x0800 /* Place busy groups earlier in the domain */
-#define SD_PREFER_SIBLING 0x1000 /* Prefer to place tasks in a sibling domain */
-#define SD_OVERLAP 0x2000 /* sched_domains of this level overlap */
-#define SD_NUMA 0x4000 /* cross-node balancing */
-
-#ifdef CONFIG_SCHED_SMT
-static inline int cpu_smt_flags(void)
-{
- return SD_SHARE_CPUCAPACITY | SD_SHARE_PKG_RESOURCES;
-}
-#endif
+ /* CLONE_CHILD_SETTID: */
+ int __user *set_child_tid;
-#ifdef CONFIG_SCHED_MC
-static inline int cpu_core_flags(void)
-{
- return SD_SHARE_PKG_RESOURCES;
-}
+ /* CLONE_CHILD_CLEARTID: */
+ int __user *clear_child_tid;
+
+ u64 utime;
+ u64 stime;
+#ifdef CONFIG_ARCH_HAS_SCALED_CPUTIME
+ u64 utimescaled;
+ u64 stimescaled;
+#endif
+ u64 gtime;
+ struct prev_cputime prev_cputime;
+#ifdef CONFIG_VIRT_CPU_ACCOUNTING_GEN
+ seqcount_t vtime_seqcount;
+ unsigned long long vtime_snap;
+ enum {
+ /* Task is sleeping or running in a CPU with VTIME inactive: */
+ VTIME_INACTIVE = 0,
+ /* Task runs in userspace in a CPU with VTIME active: */
+ VTIME_USER,
+ /* Task runs in kernelspace in a CPU with VTIME active: */
+ VTIME_SYS,
+ } vtime_snap_whence;
#endif
-#ifdef CONFIG_NUMA
-static inline int cpu_numa_flags(void)
-{
- return SD_NUMA;
-}
+#ifdef CONFIG_NO_HZ_FULL
+ atomic_t tick_dep_mask;
#endif
+ /* Context switch counts: */
+ unsigned long nvcsw;
+ unsigned long nivcsw;
-extern int arch_asym_cpu_priority(int cpu);
+ /* Monotonic time in nsecs: */
+ u64 start_time;
-struct sched_domain_attr {
- int relax_domain_level;
-};
+ /* Boot based time in nsecs: */
+ u64 real_start_time;
-#define SD_ATTR_INIT (struct sched_domain_attr) { \
- .relax_domain_level = -1, \
-}
+ /* MM fault and swap info: this can arguably be seen as either mm-specific or thread-specific: */
+ unsigned long min_flt;
+ unsigned long maj_flt;
-extern int sched_domain_level_max;
+#ifdef CONFIG_POSIX_TIMERS
+ struct task_cputime cputime_expires;
+ struct list_head cpu_timers[3];
+#endif
-struct sched_group;
+ /* Process credentials: */
-struct sched_domain_shared {
- atomic_t ref;
- atomic_t nr_busy_cpus;
- int has_idle_cores;
-};
+ /* Tracer's credentials at attach: */
+ const struct cred __rcu *ptracer_cred;
-struct sched_domain {
- /* These fields must be setup */
- struct sched_domain *parent; /* top domain must be null terminated */
- struct sched_domain *child; /* bottom domain must be null terminated */
- struct sched_group *groups; /* the balancing groups of the domain */
- unsigned long min_interval; /* Minimum balance interval ms */
- unsigned long max_interval; /* Maximum balance interval ms */
- unsigned int busy_factor; /* less balancing by factor if busy */
- unsigned int imbalance_pct; /* No balance until over watermark */
- unsigned int cache_nice_tries; /* Leave cache hot tasks for # tries */
- unsigned int busy_idx;
- unsigned int idle_idx;
- unsigned int newidle_idx;
- unsigned int wake_idx;
- unsigned int forkexec_idx;
- unsigned int smt_gain;
-
- int nohz_idle; /* NOHZ IDLE status */
- int flags; /* See SD_* */
- int level;
-
- /* Runtime fields. */
- unsigned long last_balance; /* init to jiffies. units in jiffies */
- unsigned int balance_interval; /* initialise to 1. units in ms. */
- unsigned int nr_balance_failed; /* initialise to 0 */
-
- /* idle_balance() stats */
- u64 max_newidle_lb_cost;
- unsigned long next_decay_max_lb_cost;
-
- u64 avg_scan_cost; /* select_idle_sibling */
+ /* Objective and real subjective task credentials (COW): */
+ const struct cred __rcu *real_cred;
+
+ /* Effective (overridable) subjective task credentials (COW): */
+ const struct cred __rcu *cred;
-#ifdef CONFIG_SCHEDSTATS
- /* load_balance() stats */
- unsigned int lb_count[CPU_MAX_IDLE_TYPES];
- unsigned int lb_failed[CPU_MAX_IDLE_TYPES];
- unsigned int lb_balanced[CPU_MAX_IDLE_TYPES];
- unsigned int lb_imbalance[CPU_MAX_IDLE_TYPES];
- unsigned int lb_gained[CPU_MAX_IDLE_TYPES];
- unsigned int lb_hot_gained[CPU_MAX_IDLE_TYPES];
- unsigned int lb_nobusyg[CPU_MAX_IDLE_TYPES];
- unsigned int lb_nobusyq[CPU_MAX_IDLE_TYPES];
-
- /* Active load balancing */
- unsigned int alb_count;
- unsigned int alb_failed;
- unsigned int alb_pushed;
-
- /* SD_BALANCE_EXEC stats */
- unsigned int sbe_count;
- unsigned int sbe_balanced;
- unsigned int sbe_pushed;
-
- /* SD_BALANCE_FORK stats */
- unsigned int sbf_count;
- unsigned int sbf_balanced;
- unsigned int sbf_pushed;
-
- /* try_to_wake_up() stats */
- unsigned int ttwu_wake_remote;
- unsigned int ttwu_move_affine;
- unsigned int ttwu_move_balance;
-#endif
-#ifdef CONFIG_SCHED_DEBUG
- char *name;
-#endif
- union {
- void *private; /* used during construction */
- struct rcu_head rcu; /* used during destruction */
- };
- struct sched_domain_shared *shared;
-
- unsigned int span_weight;
/*
- * Span of all CPUs in this domain.
+ * executable name, excluding path.
*
- * NOTE: this field is variable length. (Allocated dynamically
- * by attaching extra space to the end of the structure,
- * depending on how many CPUs the kernel has booted up with)
+ * - normally initialized setup_new_exec()
+ * - access it with [gs]et_task_comm()
+ * - lock it with task_lock()
*/
- unsigned long span[0];
-};
-
-static inline struct cpumask *sched_domain_span(struct sched_domain *sd)
-{
- return to_cpumask(sd->span);
-}
+ char comm[TASK_COMM_LEN];
-extern void partition_sched_domains(int ndoms_new, cpumask_var_t doms_new[],
- struct sched_domain_attr *dattr_new);
+ struct nameidata *nameidata;
-/* Allocate an array of sched domains, for partition_sched_domains(). */
-cpumask_var_t *alloc_sched_domains(unsigned int ndoms);
-void free_sched_domains(cpumask_var_t doms[], unsigned int ndoms);
+#ifdef CONFIG_SYSVIPC
+ struct sysv_sem sysvsem;
+ struct sysv_shm sysvshm;
+#endif
+#ifdef CONFIG_DETECT_HUNG_TASK
+ unsigned long last_switch_count;
+#endif
+ /* Filesystem information: */
+ struct fs_struct *fs;
-bool cpus_share_cache(int this_cpu, int that_cpu);
+ /* Open file information: */
+ struct files_struct *files;
-typedef const struct cpumask *(*sched_domain_mask_f)(int cpu);
-typedef int (*sched_domain_flags_f)(void);
+ /* Namespaces: */
+ struct nsproxy *nsproxy;
-#define SDTL_OVERLAP 0x01
+ /* Signal handlers: */
+ struct signal_struct *signal;
+ struct sighand_struct *sighand;
+ sigset_t blocked;
+ sigset_t real_blocked;
+ /* Restored if set_restore_sigmask() was used: */
+ sigset_t saved_sigmask;
+ struct sigpending pending;
+ unsigned long sas_ss_sp;
+ size_t sas_ss_size;
+ unsigned int sas_ss_flags;
-struct sd_data {
- struct sched_domain **__percpu sd;
- struct sched_domain_shared **__percpu sds;
- struct sched_group **__percpu sg;
- struct sched_group_capacity **__percpu sgc;
-};
+ struct callback_head *task_works;
-struct sched_domain_topology_level {
- sched_domain_mask_f mask;
- sched_domain_flags_f sd_flags;
- int flags;
- int numa_level;
- struct sd_data data;
-#ifdef CONFIG_SCHED_DEBUG
- char *name;
+ struct audit_context *audit_context;
+#ifdef CONFIG_AUDITSYSCALL
+ kuid_t loginuid;
+ unsigned int sessionid;
#endif
-};
+ struct seccomp seccomp;
-extern void set_sched_topology(struct sched_domain_topology_level *tl);
-extern void wake_up_if_idle(int cpu);
+ /* Thread group tracking: */
+ u32 parent_exec_id;
+ u32 self_exec_id;
-#ifdef CONFIG_SCHED_DEBUG
-# define SD_INIT_NAME(type) .name = #type
-#else
-# define SD_INIT_NAME(type)
-#endif
+ /* Protection against (de-)allocation: mm, files, fs, tty, keyrings, mems_allowed, mempolicy: */
+ spinlock_t alloc_lock;
-#else /* CONFIG_SMP */
+ /* Protection of the PI data structures: */
+ raw_spinlock_t pi_lock;
-struct sched_domain_attr;
+ struct wake_q_node wake_q;
-static inline void
-partition_sched_domains(int ndoms_new, cpumask_var_t doms_new[],
- struct sched_domain_attr *dattr_new)
-{
-}
+#ifdef CONFIG_RT_MUTEXES
+ /* PI waiters blocked on a rt_mutex held by this task: */
+ struct rb_root pi_waiters;
+ struct rb_node *pi_waiters_leftmost;
+ /* Deadlock detection and priority inheritance handling: */
+ struct rt_mutex_waiter *pi_blocked_on;
+#endif
-static inline bool cpus_share_cache(int this_cpu, int that_cpu)
-{
- return true;
-}
+#ifdef CONFIG_DEBUG_MUTEXES
+ /* Mutex deadlock detection: */
+ struct mutex_waiter *blocked_on;
+#endif
-#endif /* !CONFIG_SMP */
+#ifdef CONFIG_TRACE_IRQFLAGS
+ unsigned int irq_events;
+ unsigned long hardirq_enable_ip;
+ unsigned long hardirq_disable_ip;
+ unsigned int hardirq_enable_event;
+ unsigned int hardirq_disable_event;
+ int hardirqs_enabled;
+ int hardirq_context;
+ unsigned long softirq_disable_ip;
+ unsigned long softirq_enable_ip;
+ unsigned int softirq_disable_event;
+ unsigned int softirq_enable_event;
+ int softirqs_enabled;
+ int softirq_context;
+#endif
+
+#ifdef CONFIG_LOCKDEP
+# define MAX_LOCK_DEPTH 48UL
+ u64 curr_chain_key;
+ int lockdep_depth;
+ unsigned int lockdep_recursion;
+ struct held_lock held_locks[MAX_LOCK_DEPTH];
+ gfp_t lockdep_reclaim_gfp;
+#endif
+#ifdef CONFIG_UBSAN
+ unsigned int in_ubsan;
+#endif
-struct io_context; /* See blkdev.h */
+ /* Journalling filesystem info: */
+ void *journal_info;
+ /* Stacked block device info: */
+ struct bio_list *bio_list;
-#ifdef ARCH_HAS_PREFETCH_SWITCH_STACK
-extern void prefetch_stack(struct task_struct *t);
-#else
-static inline void prefetch_stack(struct task_struct *t) { }
+#ifdef CONFIG_BLOCK
+ /* Stack plugging: */
+ struct blk_plug *plug;
#endif
-struct audit_context; /* See audit.c */
-struct mempolicy;
-struct pipe_inode_info;
-struct uts_namespace;
+ /* VM state: */
+ struct reclaim_state *reclaim_state;
-struct load_weight {
- unsigned long weight;
- u32 inv_weight;
-};
+ struct backing_dev_info *backing_dev_info;
-/*
- * The load_avg/util_avg accumulates an infinite geometric series
- * (see __update_load_avg() in kernel/sched/fair.c).
- *
- * [load_avg definition]
- *
- * load_avg = runnable% * scale_load_down(load)
- *
- * where runnable% is the time ratio that a sched_entity is runnable.
- * For cfs_rq, it is the aggregated load_avg of all runnable and
- * blocked sched_entities.
- *
- * load_avg may also take frequency scaling into account:
- *
- * load_avg = runnable% * scale_load_down(load) * freq%
- *
- * where freq% is the CPU frequency normalized to the highest frequency.
- *
- * [util_avg definition]
- *
- * util_avg = running% * SCHED_CAPACITY_SCALE
- *
- * where running% is the time ratio that a sched_entity is running on
- * a CPU. For cfs_rq, it is the aggregated util_avg of all runnable
- * and blocked sched_entities.
- *
- * util_avg may also factor frequency scaling and CPU capacity scaling:
- *
- * util_avg = running% * SCHED_CAPACITY_SCALE * freq% * capacity%
- *
- * where freq% is the same as above, and capacity% is the CPU capacity
- * normalized to the greatest capacity (due to uarch differences, etc).
- *
- * N.B., the above ratios (runnable%, running%, freq%, and capacity%)
- * themselves are in the range of [0, 1]. To do fixed point arithmetics,
- * we therefore scale them to as large a range as necessary. This is for
- * example reflected by util_avg's SCHED_CAPACITY_SCALE.
- *
- * [Overflow issue]
- *
- * The 64-bit load_sum can have 4353082796 (=2^64/47742/88761) entities
- * with the highest load (=88761), always runnable on a single cfs_rq,
- * and should not overflow as the number already hits PID_MAX_LIMIT.
- *
- * For all other cases (including 32-bit kernels), struct load_weight's
- * weight will overflow first before we do, because:
- *
- * Max(load_avg) <= Max(load.weight)
- *
- * Then it is the load_weight's responsibility to consider overflow
- * issues.
- */
-struct sched_avg {
- u64 last_update_time, load_sum;
- u32 util_sum, period_contrib;
- unsigned long load_avg, util_avg;
-};
-
-#ifdef CONFIG_SCHEDSTATS
-struct sched_statistics {
- u64 wait_start;
- u64 wait_max;
- u64 wait_count;
- u64 wait_sum;
- u64 iowait_count;
- u64 iowait_sum;
-
- u64 sleep_start;
- u64 sleep_max;
- s64 sum_sleep_runtime;
-
- u64 block_start;
- u64 block_max;
- u64 exec_max;
- u64 slice_max;
-
- u64 nr_migrations_cold;
- u64 nr_failed_migrations_affine;
- u64 nr_failed_migrations_running;
- u64 nr_failed_migrations_hot;
- u64 nr_forced_migrations;
-
- u64 nr_wakeups;
- u64 nr_wakeups_sync;
- u64 nr_wakeups_migrate;
- u64 nr_wakeups_local;
- u64 nr_wakeups_remote;
- u64 nr_wakeups_affine;
- u64 nr_wakeups_affine_attempts;
- u64 nr_wakeups_passive;
- u64 nr_wakeups_idle;
-};
-#endif
-
-struct sched_entity {
- struct load_weight load; /* for load-balancing */
- struct rb_node run_node;
- struct list_head group_node;
- unsigned int on_rq;
-
- u64 exec_start;
- u64 sum_exec_runtime;
- u64 vruntime;
- u64 prev_sum_exec_runtime;
-
- u64 nr_migrations;
-
-#ifdef CONFIG_SCHEDSTATS
- struct sched_statistics statistics;
-#endif
-
-#ifdef CONFIG_FAIR_GROUP_SCHED
- int depth;
- struct sched_entity *parent;
- /* rq on which this entity is (to be) queued: */
- struct cfs_rq *cfs_rq;
- /* rq "owned" by this entity/group: */
- struct cfs_rq *my_q;
-#endif
-
-#ifdef CONFIG_SMP
- /*
- * Per entity load average tracking.
- *
- * Put into separate cache line so it does not
- * collide with read-mostly values above.
- */
- struct sched_avg avg ____cacheline_aligned_in_smp;
-#endif
-};
-
-struct sched_rt_entity {
- struct list_head run_list;
- unsigned long timeout;
- unsigned long watchdog_stamp;
- unsigned int time_slice;
- unsigned short on_rq;
- unsigned short on_list;
-
- struct sched_rt_entity *back;
-#ifdef CONFIG_RT_GROUP_SCHED
- struct sched_rt_entity *parent;
- /* rq on which this entity is (to be) queued: */
- struct rt_rq *rt_rq;
- /* rq "owned" by this entity/group: */
- struct rt_rq *my_q;
-#endif
-};
-
-struct sched_dl_entity {
- struct rb_node rb_node;
-
- /*
- * Original scheduling parameters. Copied here from sched_attr
- * during sched_setattr(), they will remain the same until
- * the next sched_setattr().
- */
- u64 dl_runtime; /* maximum runtime for each instance */
- u64 dl_deadline; /* relative deadline of each instance */
- u64 dl_period; /* separation of two instances (period) */
- u64 dl_bw; /* dl_runtime / dl_deadline */
-
- /*
- * Actual scheduling parameters. Initialized with the values above,
- * they are continously updated during task execution. Note that
- * the remaining runtime could be < 0 in case we are in overrun.
- */
- s64 runtime; /* remaining runtime for this instance */
- u64 deadline; /* absolute deadline for this instance */
- unsigned int flags; /* specifying the scheduler behaviour */
-
- /*
- * Some bool flags:
- *
- * @dl_throttled tells if we exhausted the runtime. If so, the
- * task has to wait for a replenishment to be performed at the
- * next firing of dl_timer.
- *
- * @dl_boosted tells if we are boosted due to DI. If so we are
- * outside bandwidth enforcement mechanism (but only until we
- * exit the critical section);
- *
- * @dl_yielded tells if task gave up the cpu before consuming
- * all its available runtime during the last job.
- */
- int dl_throttled, dl_boosted, dl_yielded;
-
- /*
- * Bandwidth enforcement timer. Each -deadline task has its
- * own bandwidth to be enforced, thus we need one timer per task.
- */
- struct hrtimer dl_timer;
-};
-
-union rcu_special {
- struct {
- u8 blocked;
- u8 need_qs;
- u8 exp_need_qs;
- u8 pad; /* Otherwise the compiler can store garbage here. */
- } b; /* Bits. */
- u32 s; /* Set of bits. */
-};
-struct rcu_node;
-
-enum perf_event_task_context {
- perf_invalid_context = -1,
- perf_hw_context = 0,
- perf_sw_context,
- perf_nr_task_contexts,
-};
-
-/* Track pages that require TLB flushes */
-struct tlbflush_unmap_batch {
- /*
- * Each bit set is a CPU that potentially has a TLB entry for one of
- * the PFNs being flushed. See set_tlb_ubc_flush_pending().
- */
- struct cpumask cpumask;
-
- /* True if any bit in cpumask is set */
- bool flush_required;
-
- /*
- * If true then the PTE was dirty when unmapped. The entry must be
- * flushed before IO is initiated or a stale TLB entry potentially
- * allows an update without redirtying the page.
- */
- bool writable;
-};
-
-struct task_struct {
-#ifdef CONFIG_THREAD_INFO_IN_TASK
- /*
- * For reasons of header soup (see current_thread_info()), this
- * must be the first element of task_struct.
- */
- struct thread_info thread_info;
-#endif
- volatile long state; /* -1 unrunnable, 0 runnable, >0 stopped */
- void *stack;
- atomic_t usage;
- unsigned int flags; /* per process flags, defined below */
- unsigned int ptrace;
-
-#ifdef CONFIG_SMP
- struct llist_node wake_entry;
- int on_cpu;
-#ifdef CONFIG_THREAD_INFO_IN_TASK
- unsigned int cpu; /* current CPU */
-#endif
- unsigned int wakee_flips;
- unsigned long wakee_flip_decay_ts;
- struct task_struct *last_wakee;
-
- int wake_cpu;
-#endif
- int on_rq;
-
- int prio, static_prio, normal_prio;
- unsigned int rt_priority;
- const struct sched_class *sched_class;
- struct sched_entity se;
- struct sched_rt_entity rt;
-#ifdef CONFIG_CGROUP_SCHED
- struct task_group *sched_task_group;
-#endif
- struct sched_dl_entity dl;
-
-#ifdef CONFIG_PREEMPT_NOTIFIERS
- /* list of struct preempt_notifier: */
- struct hlist_head preempt_notifiers;
-#endif
-
-#ifdef CONFIG_BLK_DEV_IO_TRACE
- unsigned int btrace_seq;
-#endif
-
- unsigned int policy;
- int nr_cpus_allowed;
- cpumask_t cpus_allowed;
-
-#ifdef CONFIG_PREEMPT_RCU
- int rcu_read_lock_nesting;
- union rcu_special rcu_read_unlock_special;
- struct list_head rcu_node_entry;
- struct rcu_node *rcu_blocked_node;
-#endif /* #ifdef CONFIG_PREEMPT_RCU */
-#ifdef CONFIG_TASKS_RCU
- unsigned long rcu_tasks_nvcsw;
- bool rcu_tasks_holdout;
- struct list_head rcu_tasks_holdout_list;
- int rcu_tasks_idle_cpu;
-#endif /* #ifdef CONFIG_TASKS_RCU */
-
-#ifdef CONFIG_SCHED_INFO
- struct sched_info sched_info;
-#endif
-
- struct list_head tasks;
-#ifdef CONFIG_SMP
- struct plist_node pushable_tasks;
- struct rb_node pushable_dl_tasks;
-#endif
-
- struct mm_struct *mm, *active_mm;
- /* per-thread vma caching */
- u32 vmacache_seqnum;
- struct vm_area_struct *vmacache[VMACACHE_SIZE];
-#if defined(SPLIT_RSS_COUNTING)
- struct task_rss_stat rss_stat;
-#endif
-/* task state */
- int exit_state;
- int exit_code, exit_signal;
- int pdeath_signal; /* The signal sent when the parent dies */
- unsigned long jobctl; /* JOBCTL_*, siglock protected */
-
- /* Used for emulating ABI behavior of previous Linux versions */
- unsigned int personality;
-
- /* scheduler bits, serialized by scheduler locks */
- unsigned sched_reset_on_fork:1;
- unsigned sched_contributes_to_load:1;
- unsigned sched_migrated:1;
- unsigned sched_remote_wakeup:1;
- unsigned :0; /* force alignment to the next boundary */
-
- /* unserialized, strictly 'current' */
- unsigned in_execve:1; /* bit to tell LSMs we're in execve */
- unsigned in_iowait:1;
-#if !defined(TIF_RESTORE_SIGMASK)
- unsigned restore_sigmask:1;
-#endif
-#ifdef CONFIG_MEMCG
- unsigned memcg_may_oom:1;
-#ifndef CONFIG_SLOB
- unsigned memcg_kmem_skip_account:1;
-#endif
-#endif
-#ifdef CONFIG_COMPAT_BRK
- unsigned brk_randomized:1;
-#endif
-
- unsigned long atomic_flags; /* Flags needing atomic access. */
-
- struct restart_block restart_block;
-
- pid_t pid;
- pid_t tgid;
-
-#ifdef CONFIG_CC_STACKPROTECTOR
- /* Canary value for the -fstack-protector gcc feature */
- unsigned long stack_canary;
-#endif
- /*
- * pointers to (original) parent process, youngest child, younger sibling,
- * older sibling, respectively. (p->father can be replaced with
- * p->real_parent->pid)
- */
- struct task_struct __rcu *real_parent; /* real parent process */
- struct task_struct __rcu *parent; /* recipient of SIGCHLD, wait4() reports */
- /*
- * children/sibling forms the list of my natural children
- */
- struct list_head children; /* list of my children */
- struct list_head sibling; /* linkage in my parent's children list */
- struct task_struct *group_leader; /* threadgroup leader */
-
- /*
- * ptraced is the list of tasks this task is using ptrace on.
- * This includes both natural children and PTRACE_ATTACH targets.
- * p->ptrace_entry is p's link on the p->parent->ptraced list.
- */
- struct list_head ptraced;
- struct list_head ptrace_entry;
-
- /* PID/PID hash table linkage. */
- struct pid_link pids[PIDTYPE_MAX];
- struct list_head thread_group;
- struct list_head thread_node;
-
- struct completion *vfork_done; /* for vfork() */
- int __user *set_child_tid; /* CLONE_CHILD_SETTID */
- int __user *clear_child_tid; /* CLONE_CHILD_CLEARTID */
-
- u64 utime, stime;
-#ifdef CONFIG_ARCH_HAS_SCALED_CPUTIME
- u64 utimescaled, stimescaled;
-#endif
- u64 gtime;
- struct prev_cputime prev_cputime;
-#ifdef CONFIG_VIRT_CPU_ACCOUNTING_GEN
- seqcount_t vtime_seqcount;
- unsigned long long vtime_snap;
- enum {
- /* Task is sleeping or running in a CPU with VTIME inactive */
- VTIME_INACTIVE = 0,
- /* Task runs in userspace in a CPU with VTIME active */
- VTIME_USER,
- /* Task runs in kernelspace in a CPU with VTIME active */
- VTIME_SYS,
- } vtime_snap_whence;
-#endif
-
-#ifdef CONFIG_NO_HZ_FULL
- atomic_t tick_dep_mask;
-#endif
- unsigned long nvcsw, nivcsw; /* context switch counts */
- u64 start_time; /* monotonic time in nsec */
- u64 real_start_time; /* boot based time in nsec */
-/* mm fault and swap info: this can arguably be seen as either mm-specific or thread-specific */
- unsigned long min_flt, maj_flt;
-
-#ifdef CONFIG_POSIX_TIMERS
- struct task_cputime cputime_expires;
- struct list_head cpu_timers[3];
-#endif
-
-/* process credentials */
- const struct cred __rcu *ptracer_cred; /* Tracer's credentials at attach */
- const struct cred __rcu *real_cred; /* objective and real subjective task
- * credentials (COW) */
- const struct cred __rcu *cred; /* effective (overridable) subjective task
- * credentials (COW) */
- char comm[TASK_COMM_LEN]; /* executable name excluding path
- - access with [gs]et_task_comm (which lock
- it with task_lock())
- - initialized normally by setup_new_exec */
-/* file system info */
- struct nameidata *nameidata;
-#ifdef CONFIG_SYSVIPC
-/* ipc stuff */
- struct sysv_sem sysvsem;
- struct sysv_shm sysvshm;
-#endif
-#ifdef CONFIG_DETECT_HUNG_TASK
-/* hung task detection */
- unsigned long last_switch_count;
-#endif
-/* filesystem information */
- struct fs_struct *fs;
-/* open file information */
- struct files_struct *files;
-/* namespaces */
- struct nsproxy *nsproxy;
-/* signal handlers */
- struct signal_struct *signal;
- struct sighand_struct *sighand;
-
- sigset_t blocked, real_blocked;
- sigset_t saved_sigmask; /* restored if set_restore_sigmask() was used */
- struct sigpending pending;
-
- unsigned long sas_ss_sp;
- size_t sas_ss_size;
- unsigned sas_ss_flags;
-
- struct callback_head *task_works;
-
- struct audit_context *audit_context;
-#ifdef CONFIG_AUDITSYSCALL
- kuid_t loginuid;
- unsigned int sessionid;
-#endif
- struct seccomp seccomp;
-
-/* Thread group tracking */
- u32 parent_exec_id;
- u32 self_exec_id;
-/* Protection of (de-)allocation: mm, files, fs, tty, keyrings, mems_allowed,
- * mempolicy */
- spinlock_t alloc_lock;
-
- /* Protection of the PI data structures: */
- raw_spinlock_t pi_lock;
-
- struct wake_q_node wake_q;
-
-#ifdef CONFIG_RT_MUTEXES
- /* PI waiters blocked on a rt_mutex held by this task */
- struct rb_root pi_waiters;
- struct rb_node *pi_waiters_leftmost;
- /* Deadlock detection and priority inheritance handling */
- struct rt_mutex_waiter *pi_blocked_on;
-#endif
-
-#ifdef CONFIG_DEBUG_MUTEXES
- /* mutex deadlock detection */
- struct mutex_waiter *blocked_on;
-#endif
-#ifdef CONFIG_TRACE_IRQFLAGS
- unsigned int irq_events;
- unsigned long hardirq_enable_ip;
- unsigned long hardirq_disable_ip;
- unsigned int hardirq_enable_event;
- unsigned int hardirq_disable_event;
- int hardirqs_enabled;
- int hardirq_context;
- unsigned long softirq_disable_ip;
- unsigned long softirq_enable_ip;
- unsigned int softirq_disable_event;
- unsigned int softirq_enable_event;
- int softirqs_enabled;
- int softirq_context;
-#endif
-#ifdef CONFIG_LOCKDEP
-# define MAX_LOCK_DEPTH 48UL
- u64 curr_chain_key;
- int lockdep_depth;
- unsigned int lockdep_recursion;
- struct held_lock held_locks[MAX_LOCK_DEPTH];
- gfp_t lockdep_reclaim_gfp;
-#endif
-#ifdef CONFIG_UBSAN
- unsigned int in_ubsan;
-#endif
-
-/* journalling filesystem info */
- void *journal_info;
-
-/* stacked block device info */
- struct bio_list *bio_list;
-
-#ifdef CONFIG_BLOCK
-/* stack plugging */
- struct blk_plug *plug;
-#endif
-
-/* VM state */
- struct reclaim_state *reclaim_state;
+ struct io_context *io_context;
- struct backing_dev_info *backing_dev_info;
+ /* Ptrace state: */
+ unsigned long ptrace_message;
+ siginfo_t *last_siginfo;
- struct io_context *io_context;
-
- unsigned long ptrace_message;
- siginfo_t *last_siginfo; /* For ptrace use. */
- struct task_io_accounting ioac;
-#if defined(CONFIG_TASK_XACCT)
- u64 acct_rss_mem1; /* accumulated rss usage */
- u64 acct_vm_mem1; /* accumulated virtual memory usage */
- u64 acct_timexpd; /* stime + utime since last update */
+ struct task_io_accounting ioac;
+#ifdef CONFIG_TASK_XACCT
+ /* Accumulated RSS usage: */
+ u64 acct_rss_mem1;
+ /* Accumulated virtual memory usage: */
+ u64 acct_vm_mem1;
+ /* stime + utime since last update: */
+ u64 acct_timexpd;
#endif
#ifdef CONFIG_CPUSETS
- nodemask_t mems_allowed; /* Protected by alloc_lock */
- seqcount_t mems_allowed_seq; /* Seqence no to catch updates */
- int cpuset_mem_spread_rotor;
- int cpuset_slab_spread_rotor;
+ /* Protected by ->alloc_lock: */
+ nodemask_t mems_allowed;
+ /* Seqence number to catch updates: */
+ seqcount_t mems_allowed_seq;
+ int cpuset_mem_spread_rotor;
+ int cpuset_slab_spread_rotor;
#endif
#ifdef CONFIG_CGROUPS
- /* Control Group info protected by css_set_lock */
- struct css_set __rcu *cgroups;
- /* cg_list protected by css_set_lock and tsk->alloc_lock */
- struct list_head cg_list;
+ /* Control Group info protected by css_set_lock: */
+ struct css_set __rcu *cgroups;
+ /* cg_list protected by css_set_lock and tsk->alloc_lock: */
+ struct list_head cg_list;
#endif
#ifdef CONFIG_INTEL_RDT_A
- int closid;
+ int closid;
#endif
#ifdef CONFIG_FUTEX
- struct robust_list_head __user *robust_list;
+ struct robust_list_head __user *robust_list;
#ifdef CONFIG_COMPAT
struct compat_robust_list_head __user *compat_robust_list;
#endif
- struct list_head pi_state_list;
- struct futex_pi_state *pi_state_cache;
+ struct list_head pi_state_list;
+ struct futex_pi_state *pi_state_cache;
#endif
#ifdef CONFIG_PERF_EVENTS
- struct perf_event_context *perf_event_ctxp[perf_nr_task_contexts];
- struct mutex perf_event_mutex;
- struct list_head perf_event_list;
+ struct perf_event_context *perf_event_ctxp[perf_nr_task_contexts];
+ struct mutex perf_event_mutex;
+ struct list_head perf_event_list;
#endif
#ifdef CONFIG_DEBUG_PREEMPT
- unsigned long preempt_disable_ip;
+ unsigned long preempt_disable_ip;
#endif
#ifdef CONFIG_NUMA
- struct mempolicy *mempolicy; /* Protected by alloc_lock */
- short il_next;
- short pref_node_fork;
+ /* Protected by alloc_lock: */
+ struct mempolicy *mempolicy;
+ short il_next;
+ short pref_node_fork;
#endif
#ifdef CONFIG_NUMA_BALANCING
- int numa_scan_seq;
- unsigned int numa_scan_period;
- unsigned int numa_scan_period_max;
- int numa_preferred_nid;
- unsigned long numa_migrate_retry;
- u64 node_stamp; /* migration stamp */
- u64 last_task_numa_placement;
- u64 last_sum_exec_runtime;
- struct callback_head numa_work;
-
- struct list_head numa_entry;
- struct numa_group *numa_group;
-
- /*
- * numa_faults is an array split into four regions:
- * faults_memory, faults_cpu, faults_memory_buffer, faults_cpu_buffer
- * in this precise order.
- *
- * faults_memory: Exponential decaying average of faults on a per-node
- * basis. Scheduling placement decisions are made based on these
- * counts. The values remain static for the duration of a PTE scan.
- * faults_cpu: Track the nodes the process was running on when a NUMA
- * hinting fault was incurred.
- * faults_memory_buffer and faults_cpu_buffer: Record faults per node
- * during the current scan window. When the scan completes, the counts
- * in faults_memory and faults_cpu decay and these values are copied.
- */
- unsigned long *numa_faults;
- unsigned long total_numa_faults;
-
- /*
- * numa_faults_locality tracks if faults recorded during the last
- * scan window were remote/local or failed to migrate. The task scan
- * period is adapted based on the locality of the faults with different
- * weights depending on whether they were shared or private faults
- */
- unsigned long numa_faults_locality[3];
+ int numa_scan_seq;
+ unsigned int numa_scan_period;
+ unsigned int numa_scan_period_max;
+ int numa_preferred_nid;
+ unsigned long numa_migrate_retry;
+ /* Migration stamp: */
+ u64 node_stamp;
+ u64 last_task_numa_placement;
+ u64 last_sum_exec_runtime;
+ struct callback_head numa_work;
+
+ struct list_head numa_entry;
+ struct numa_group *numa_group;
- unsigned long numa_pages_migrated;
-#endif /* CONFIG_NUMA_BALANCING */
-
-#ifdef CONFIG_ARCH_WANT_BATCHED_UNMAP_TLB_FLUSH
- struct tlbflush_unmap_batch tlb_ubc;
-#endif
-
- struct rcu_head rcu;
-
- /*
- * cache last used pipe for splice
- */
- struct pipe_inode_info *splice_pipe;
-
- struct page_frag task_frag;
-
-#ifdef CONFIG_TASK_DELAY_ACCT
- struct task_delay_info *delays;
-#endif
-#ifdef CONFIG_FAULT_INJECTION
- int make_it_fail;
-#endif
- /*
- * when (nr_dirtied >= nr_dirtied_pause), it's time to call
- * balance_dirty_pages() for some dirty throttling pause
- */
- int nr_dirtied;
- int nr_dirtied_pause;
- unsigned long dirty_paused_when; /* start of a write-and-pause period */
-
-#ifdef CONFIG_LATENCYTOP
- int latency_record_count;
- struct latency_record latency_record[LT_SAVECOUNT];
-#endif
- /*
- * time slack values; these are used to round up poll() and
- * select() etc timeout values. These are in nanoseconds.
- */
- u64 timer_slack_ns;
- u64 default_timer_slack_ns;
-
-#ifdef CONFIG_KASAN
- unsigned int kasan_depth;
-#endif
-#ifdef CONFIG_FUNCTION_GRAPH_TRACER
- /* Index of current stored address in ret_stack */
- int curr_ret_stack;
- /* Stack of return addresses for return function tracing */
- struct ftrace_ret_stack *ret_stack;
- /* time stamp for last schedule */
- unsigned long long ftrace_timestamp;
- /*
- * Number of functions that haven't been traced
- * because of depth overrun.
- */
- atomic_t trace_overrun;
- /* Pause for the tracing */
- atomic_t tracing_graph_pause;
-#endif
-#ifdef CONFIG_TRACING
- /* state flags for use by tracers */
- unsigned long trace;
- /* bitmask and counter of trace recursion */
- unsigned long trace_recursion;
-#endif /* CONFIG_TRACING */
-#ifdef CONFIG_KCOV
- /* Coverage collection mode enabled for this task (0 if disabled). */
- enum kcov_mode kcov_mode;
- /* Size of the kcov_area. */
- unsigned kcov_size;
- /* Buffer for coverage collection. */
- void *kcov_area;
- /* kcov desciptor wired with this task or NULL. */
- struct kcov *kcov;
-#endif
-#ifdef CONFIG_MEMCG
- struct mem_cgroup *memcg_in_oom;
- gfp_t memcg_oom_gfp_mask;
- int memcg_oom_order;
-
- /* number of pages to reclaim on returning to userland */
- unsigned int memcg_nr_pages_over_high;
-#endif
-#ifdef CONFIG_UPROBES
- struct uprobe_task *utask;
-#endif
-#if defined(CONFIG_BCACHE) || defined(CONFIG_BCACHE_MODULE)
- unsigned int sequential_io;
- unsigned int sequential_io_avg;
-#endif
-#ifdef CONFIG_DEBUG_ATOMIC_SLEEP
- unsigned long task_state_change;
-#endif
- int pagefault_disabled;
-#ifdef CONFIG_MMU
- struct task_struct *oom_reaper_list;
-#endif
-#ifdef CONFIG_VMAP_STACK
- struct vm_struct *stack_vm_area;
-#endif
-#ifdef CONFIG_THREAD_INFO_IN_TASK
- /* A live task holds one reference. */
- atomic_t stack_refcount;
-#endif
-/* CPU-specific state of this task */
- struct thread_struct thread;
-/*
- * WARNING: on x86, 'thread_struct' contains a variable-sized
- * structure. It *MUST* be at the end of 'task_struct'.
- *
- * Do not put anything below here!
- */
-};
-
-#ifdef CONFIG_ARCH_WANTS_DYNAMIC_TASK_STRUCT
-extern int arch_task_struct_size __read_mostly;
-#else
-# define arch_task_struct_size (sizeof(struct task_struct))
-#endif
-
-#ifdef CONFIG_VMAP_STACK
-static inline struct vm_struct *task_stack_vm_area(const struct task_struct *t)
-{
- return t->stack_vm_area;
-}
-#else
-static inline struct vm_struct *task_stack_vm_area(const struct task_struct *t)
-{
- return NULL;
-}
-#endif
-
-/* Future-safe accessor for struct task_struct's cpus_allowed. */
-#define tsk_cpus_allowed(tsk) (&(tsk)->cpus_allowed)
-
-static inline int tsk_nr_cpus_allowed(struct task_struct *p)
-{
- return p->nr_cpus_allowed;
-}
-
-#define TNF_MIGRATED 0x01
-#define TNF_NO_GROUP 0x02
-#define TNF_SHARED 0x04
-#define TNF_FAULT_LOCAL 0x08
-#define TNF_MIGRATE_FAIL 0x10
-
-static inline bool in_vfork(struct task_struct *tsk)
-{
- bool ret;
-
- /*
- * need RCU to access ->real_parent if CLONE_VM was used along with
- * CLONE_PARENT.
- *
- * We check real_parent->mm == tsk->mm because CLONE_VFORK does not
- * imply CLONE_VM
- *
- * CLONE_VFORK can be used with CLONE_PARENT/CLONE_THREAD and thus
- * ->real_parent is not necessarily the task doing vfork(), so in
- * theory we can't rely on task_lock() if we want to dereference it.
- *
- * And in this case we can't trust the real_parent->mm == tsk->mm
- * check, it can be false negative. But we do not care, if init or
- * another oom-unkillable task does this it should blame itself.
- */
- rcu_read_lock();
- ret = tsk->vfork_done && tsk->real_parent->mm == tsk->mm;
- rcu_read_unlock();
-
- return ret;
-}
-
-#ifdef CONFIG_NUMA_BALANCING
-extern void task_numa_fault(int last_node, int node, int pages, int flags);
-extern pid_t task_numa_group_id(struct task_struct *p);
-extern void set_numabalancing_state(bool enabled);
-extern void task_numa_free(struct task_struct *p);
-extern bool should_numa_migrate_memory(struct task_struct *p, struct page *page,
- int src_nid, int dst_cpu);
-#else
-static inline void task_numa_fault(int last_node, int node, int pages,
- int flags)
-{
-}
-static inline pid_t task_numa_group_id(struct task_struct *p)
-{
- return 0;
-}
-static inline void set_numabalancing_state(bool enabled)
-{
-}
-static inline void task_numa_free(struct task_struct *p)
-{
-}
-static inline bool should_numa_migrate_memory(struct task_struct *p,
- struct page *page, int src_nid, int dst_cpu)
-{
- return true;
-}
-#endif
-
-static inline struct pid *task_pid(struct task_struct *task)
-{
- return task->pids[PIDTYPE_PID].pid;
-}
-
-static inline struct pid *task_tgid(struct task_struct *task)
-{
- return task->group_leader->pids[PIDTYPE_PID].pid;
-}
-
-/*
- * Without tasklist or rcu lock it is not safe to dereference
- * the result of task_pgrp/task_session even if task == current,
- * we can race with another thread doing sys_setsid/sys_setpgid.
- */
-static inline struct pid *task_pgrp(struct task_struct *task)
-{
- return task->group_leader->pids[PIDTYPE_PGID].pid;
-}
-
-static inline struct pid *task_session(struct task_struct *task)
-{
- return task->group_leader->pids[PIDTYPE_SID].pid;
-}
-
-struct pid_namespace;
-
-/*
- * the helpers to get the task's different pids as they are seen
- * from various namespaces
- *
- * task_xid_nr() : global id, i.e. the id seen from the init namespace;
- * task_xid_vnr() : virtual id, i.e. the id seen from the pid namespace of
- * current.
- * task_xid_nr_ns() : id seen from the ns specified;
- *
- * set_task_vxid() : assigns a virtual id to a task;
- *
- * see also pid_nr() etc in include/linux/pid.h
- */
-pid_t __task_pid_nr_ns(struct task_struct *task, enum pid_type type,
- struct pid_namespace *ns);
-
-static inline pid_t task_pid_nr(struct task_struct *tsk)
-{
- return tsk->pid;
-}
-
-static inline pid_t task_pid_nr_ns(struct task_struct *tsk,
- struct pid_namespace *ns)
-{
- return __task_pid_nr_ns(tsk, PIDTYPE_PID, ns);
-}
-
-static inline pid_t task_pid_vnr(struct task_struct *tsk)
-{
- return __task_pid_nr_ns(tsk, PIDTYPE_PID, NULL);
-}
-
-
-static inline pid_t task_tgid_nr(struct task_struct *tsk)
-{
- return tsk->tgid;
-}
-
-pid_t task_tgid_nr_ns(struct task_struct *tsk, struct pid_namespace *ns);
-
-static inline pid_t task_tgid_vnr(struct task_struct *tsk)
-{
- return pid_vnr(task_tgid(tsk));
-}
-
-
-static inline int pid_alive(const struct task_struct *p);
-static inline pid_t task_ppid_nr_ns(const struct task_struct *tsk, struct pid_namespace *ns)
-{
- pid_t pid = 0;
-
- rcu_read_lock();
- if (pid_alive(tsk))
- pid = task_tgid_nr_ns(rcu_dereference(tsk->real_parent), ns);
- rcu_read_unlock();
-
- return pid;
-}
-
-static inline pid_t task_ppid_nr(const struct task_struct *tsk)
-{
- return task_ppid_nr_ns(tsk, &init_pid_ns);
-}
-
-static inline pid_t task_pgrp_nr_ns(struct task_struct *tsk,
- struct pid_namespace *ns)
-{
- return __task_pid_nr_ns(tsk, PIDTYPE_PGID, ns);
-}
-
-static inline pid_t task_pgrp_vnr(struct task_struct *tsk)
-{
- return __task_pid_nr_ns(tsk, PIDTYPE_PGID, NULL);
-}
-
-
-static inline pid_t task_session_nr_ns(struct task_struct *tsk,
- struct pid_namespace *ns)
-{
- return __task_pid_nr_ns(tsk, PIDTYPE_SID, ns);
-}
-
-static inline pid_t task_session_vnr(struct task_struct *tsk)
-{
- return __task_pid_nr_ns(tsk, PIDTYPE_SID, NULL);
-}
-
-/* obsolete, do not use */
-static inline pid_t task_pgrp_nr(struct task_struct *tsk)
-{
- return task_pgrp_nr_ns(tsk, &init_pid_ns);
-}
-
-/**
- * pid_alive - check that a task structure is not stale
- * @p: Task structure to be checked.
- *
- * Test if a process is not yet dead (at most zombie state)
- * If pid_alive fails, then pointers within the task structure
- * can be stale and must not be dereferenced.
- *
- * Return: 1 if the process is alive. 0 otherwise.
- */
-static inline int pid_alive(const struct task_struct *p)
-{
- return p->pids[PIDTYPE_PID].pid != NULL;
-}
-
-/**
- * is_global_init - check if a task structure is init. Since init
- * is free to have sub-threads we need to check tgid.
- * @tsk: Task structure to be checked.
- *
- * Check if a task structure is the first user space task the kernel created.
- *
- * Return: 1 if the task structure is init. 0 otherwise.
- */
-static inline int is_global_init(struct task_struct *tsk)
-{
- return task_tgid_nr(tsk) == 1;
-}
-
-extern struct pid *cad_pid;
-
-extern void free_task(struct task_struct *tsk);
-#define get_task_struct(tsk) do { atomic_inc(&(tsk)->usage); } while(0)
-
-extern void __put_task_struct(struct task_struct *t);
-
-static inline void put_task_struct(struct task_struct *t)
-{
- if (atomic_dec_and_test(&t->usage))
- __put_task_struct(t);
-}
-
-struct task_struct *task_rcu_dereference(struct task_struct **ptask);
-struct task_struct *try_get_task_struct(struct task_struct **ptask);
-
-#ifdef CONFIG_VIRT_CPU_ACCOUNTING_GEN
-extern void task_cputime(struct task_struct *t,
- u64 *utime, u64 *stime);
-extern u64 task_gtime(struct task_struct *t);
-#else
-static inline void task_cputime(struct task_struct *t,
- u64 *utime, u64 *stime)
-{
- *utime = t->utime;
- *stime = t->stime;
-}
-
-static inline u64 task_gtime(struct task_struct *t)
-{
- return t->gtime;
-}
-#endif
-
-#ifdef CONFIG_ARCH_HAS_SCALED_CPUTIME
-static inline void task_cputime_scaled(struct task_struct *t,
- u64 *utimescaled,
- u64 *stimescaled)
-{
- *utimescaled = t->utimescaled;
- *stimescaled = t->stimescaled;
-}
-#else
-static inline void task_cputime_scaled(struct task_struct *t,
- u64 *utimescaled,
- u64 *stimescaled)
-{
- task_cputime(t, utimescaled, stimescaled);
-}
-#endif
-
-extern void task_cputime_adjusted(struct task_struct *p, u64 *ut, u64 *st);
-extern void thread_group_cputime_adjusted(struct task_struct *p, u64 *ut, u64 *st);
-
-/*
- * Per process flags
- */
-#define PF_IDLE 0x00000002 /* I am an IDLE thread */
-#define PF_EXITING 0x00000004 /* getting shut down */
-#define PF_EXITPIDONE 0x00000008 /* pi exit done on shut down */
-#define PF_VCPU 0x00000010 /* I'm a virtual CPU */
-#define PF_WQ_WORKER 0x00000020 /* I'm a workqueue worker */
-#define PF_FORKNOEXEC 0x00000040 /* forked but didn't exec */
-#define PF_MCE_PROCESS 0x00000080 /* process policy on mce errors */
-#define PF_SUPERPRIV 0x00000100 /* used super-user privileges */
-#define PF_DUMPCORE 0x00000200 /* dumped core */
-#define PF_SIGNALED 0x00000400 /* killed by a signal */
-#define PF_MEMALLOC 0x00000800 /* Allocating memory */
-#define PF_NPROC_EXCEEDED 0x00001000 /* set_user noticed that RLIMIT_NPROC was exceeded */
-#define PF_USED_MATH 0x00002000 /* if unset the fpu must be initialized before use */
-#define PF_USED_ASYNC 0x00004000 /* used async_schedule*(), used by module init */
-#define PF_NOFREEZE 0x00008000 /* this thread should not be frozen */
-#define PF_FROZEN 0x00010000 /* frozen for system suspend */
-#define PF_FSTRANS 0x00020000 /* inside a filesystem transaction */
-#define PF_KSWAPD 0x00040000 /* I am kswapd */
-#define PF_MEMALLOC_NOIO 0x00080000 /* Allocating memory without IO involved */
-#define PF_LESS_THROTTLE 0x00100000 /* Throttle me less: I clean memory */
-#define PF_KTHREAD 0x00200000 /* I am a kernel thread */
-#define PF_RANDOMIZE 0x00400000 /* randomize virtual address space */
-#define PF_SWAPWRITE 0x00800000 /* Allowed to write to swap */
-#define PF_NO_SETAFFINITY 0x04000000 /* Userland is not allowed to meddle with cpus_allowed */
-#define PF_MCE_EARLY 0x08000000 /* Early kill for mce process policy */
-#define PF_MUTEX_TESTER 0x20000000 /* Thread belongs to the rt mutex tester */
-#define PF_FREEZER_SKIP 0x40000000 /* Freezer should not count it as freezable */
-#define PF_SUSPEND_TASK 0x80000000 /* this thread called freeze_processes and should not be frozen */
-
-/*
- * Only the _current_ task can read/write to tsk->flags, but other
- * tasks can access tsk->flags in readonly mode for example
- * with tsk_used_math (like during threaded core dumping).
- * There is however an exception to this rule during ptrace
- * or during fork: the ptracer task is allowed to write to the
- * child->flags of its traced child (same goes for fork, the parent
- * can write to the child->flags), because we're guaranteed the
- * child is not running and in turn not changing child->flags
- * at the same time the parent does it.
- */
-#define clear_stopped_child_used_math(child) do { (child)->flags &= ~PF_USED_MATH; } while (0)
-#define set_stopped_child_used_math(child) do { (child)->flags |= PF_USED_MATH; } while (0)
-#define clear_used_math() clear_stopped_child_used_math(current)
-#define set_used_math() set_stopped_child_used_math(current)
-#define conditional_stopped_child_used_math(condition, child) \
- do { (child)->flags &= ~PF_USED_MATH, (child)->flags |= (condition) ? PF_USED_MATH : 0; } while (0)
-#define conditional_used_math(condition) \
- conditional_stopped_child_used_math(condition, current)
-#define copy_to_stopped_child_used_math(child) \
- do { (child)->flags &= ~PF_USED_MATH, (child)->flags |= current->flags & PF_USED_MATH; } while (0)
-/* NOTE: this will return 0 or PF_USED_MATH, it will never return 1 */
-#define tsk_used_math(p) ((p)->flags & PF_USED_MATH)
-#define used_math() tsk_used_math(current)
-
-/* __GFP_IO isn't allowed if PF_MEMALLOC_NOIO is set in current->flags
- * __GFP_FS is also cleared as it implies __GFP_IO.
- */
-static inline gfp_t memalloc_noio_flags(gfp_t flags)
-{
- if (unlikely(current->flags & PF_MEMALLOC_NOIO))
- flags &= ~(__GFP_IO | __GFP_FS);
- return flags;
-}
-
-static inline unsigned int memalloc_noio_save(void)
-{
- unsigned int flags = current->flags & PF_MEMALLOC_NOIO;
- current->flags |= PF_MEMALLOC_NOIO;
- return flags;
-}
-
-static inline void memalloc_noio_restore(unsigned int flags)
-{
- current->flags = (current->flags & ~PF_MEMALLOC_NOIO) | flags;
-}
-
-/* Per-process atomic flags. */
-#define PFA_NO_NEW_PRIVS 0 /* May not gain new privileges. */
-#define PFA_SPREAD_PAGE 1 /* Spread page cache over cpuset */
-#define PFA_SPREAD_SLAB 2 /* Spread some slab caches over cpuset */
-#define PFA_LMK_WAITING 3 /* Lowmemorykiller is waiting */
-
-
-#define TASK_PFA_TEST(name, func) \
- static inline bool task_##func(struct task_struct *p) \
- { return test_bit(PFA_##name, &p->atomic_flags); }
-#define TASK_PFA_SET(name, func) \
- static inline void task_set_##func(struct task_struct *p) \
- { set_bit(PFA_##name, &p->atomic_flags); }
-#define TASK_PFA_CLEAR(name, func) \
- static inline void task_clear_##func(struct task_struct *p) \
- { clear_bit(PFA_##name, &p->atomic_flags); }
-
-TASK_PFA_TEST(NO_NEW_PRIVS, no_new_privs)
-TASK_PFA_SET(NO_NEW_PRIVS, no_new_privs)
-
-TASK_PFA_TEST(SPREAD_PAGE, spread_page)
-TASK_PFA_SET(SPREAD_PAGE, spread_page)
-TASK_PFA_CLEAR(SPREAD_PAGE, spread_page)
-
-TASK_PFA_TEST(SPREAD_SLAB, spread_slab)
-TASK_PFA_SET(SPREAD_SLAB, spread_slab)
-TASK_PFA_CLEAR(SPREAD_SLAB, spread_slab)
-
-TASK_PFA_TEST(LMK_WAITING, lmk_waiting)
-TASK_PFA_SET(LMK_WAITING, lmk_waiting)
-
-/*
- * task->jobctl flags
- */
-#define JOBCTL_STOP_SIGMASK 0xffff /* signr of the last group stop */
-
-#define JOBCTL_STOP_DEQUEUED_BIT 16 /* stop signal dequeued */
-#define JOBCTL_STOP_PENDING_BIT 17 /* task should stop for group stop */
-#define JOBCTL_STOP_CONSUME_BIT 18 /* consume group stop count */
-#define JOBCTL_TRAP_STOP_BIT 19 /* trap for STOP */
-#define JOBCTL_TRAP_NOTIFY_BIT 20 /* trap for NOTIFY */
-#define JOBCTL_TRAPPING_BIT 21 /* switching to TRACED */
-#define JOBCTL_LISTENING_BIT 22 /* ptracer is listening for events */
-
-#define JOBCTL_STOP_DEQUEUED (1UL << JOBCTL_STOP_DEQUEUED_BIT)
-#define JOBCTL_STOP_PENDING (1UL << JOBCTL_STOP_PENDING_BIT)
-#define JOBCTL_STOP_CONSUME (1UL << JOBCTL_STOP_CONSUME_BIT)
-#define JOBCTL_TRAP_STOP (1UL << JOBCTL_TRAP_STOP_BIT)
-#define JOBCTL_TRAP_NOTIFY (1UL << JOBCTL_TRAP_NOTIFY_BIT)
-#define JOBCTL_TRAPPING (1UL << JOBCTL_TRAPPING_BIT)
-#define JOBCTL_LISTENING (1UL << JOBCTL_LISTENING_BIT)
-
-#define JOBCTL_TRAP_MASK (JOBCTL_TRAP_STOP | JOBCTL_TRAP_NOTIFY)
-#define JOBCTL_PENDING_MASK (JOBCTL_STOP_PENDING | JOBCTL_TRAP_MASK)
-
-extern bool task_set_jobctl_pending(struct task_struct *task,
- unsigned long mask);
-extern void task_clear_jobctl_trapping(struct task_struct *task);
-extern void task_clear_jobctl_pending(struct task_struct *task,
- unsigned long mask);
-
-static inline void rcu_copy_process(struct task_struct *p)
-{
-#ifdef CONFIG_PREEMPT_RCU
- p->rcu_read_lock_nesting = 0;
- p->rcu_read_unlock_special.s = 0;
- p->rcu_blocked_node = NULL;
- INIT_LIST_HEAD(&p->rcu_node_entry);
-#endif /* #ifdef CONFIG_PREEMPT_RCU */
-#ifdef CONFIG_TASKS_RCU
- p->rcu_tasks_holdout = false;
- INIT_LIST_HEAD(&p->rcu_tasks_holdout_list);
- p->rcu_tasks_idle_cpu = -1;
-#endif /* #ifdef CONFIG_TASKS_RCU */
-}
-
-static inline void tsk_restore_flags(struct task_struct *task,
- unsigned long orig_flags, unsigned long flags)
-{
- task->flags &= ~flags;
- task->flags |= orig_flags & flags;
-}
-
-extern int cpuset_cpumask_can_shrink(const struct cpumask *cur,
- const struct cpumask *trial);
-extern int task_can_attach(struct task_struct *p,
- const struct cpumask *cs_cpus_allowed);
-#ifdef CONFIG_SMP
-extern void do_set_cpus_allowed(struct task_struct *p,
- const struct cpumask *new_mask);
-
-extern int set_cpus_allowed_ptr(struct task_struct *p,
- const struct cpumask *new_mask);
-#else
-static inline void do_set_cpus_allowed(struct task_struct *p,
- const struct cpumask *new_mask)
-{
-}
-static inline int set_cpus_allowed_ptr(struct task_struct *p,
- const struct cpumask *new_mask)
-{
- if (!cpumask_test_cpu(0, new_mask))
- return -EINVAL;
- return 0;
-}
-#endif
-
-#ifdef CONFIG_NO_HZ_COMMON
-void calc_load_enter_idle(void);
-void calc_load_exit_idle(void);
-#else
-static inline void calc_load_enter_idle(void) { }
-static inline void calc_load_exit_idle(void) { }
-#endif /* CONFIG_NO_HZ_COMMON */
-
-#ifndef cpu_relax_yield
-#define cpu_relax_yield() cpu_relax()
-#endif
-
-/*
- * Do not use outside of architecture code which knows its limitations.
- *
- * sched_clock() has no promise of monotonicity or bounded drift between
- * CPUs, use (which you should not) requires disabling IRQs.
- *
- * Please use one of the three interfaces below.
- */
-extern unsigned long long notrace sched_clock(void);
-/*
- * See the comment in kernel/sched/clock.c
- */
-extern u64 running_clock(void);
-extern u64 sched_clock_cpu(int cpu);
-
-
-extern void sched_clock_init(void);
-
-#ifndef CONFIG_HAVE_UNSTABLE_SCHED_CLOCK
-static inline void sched_clock_init_late(void)
-{
-}
-
-static inline void sched_clock_tick(void)
-{
-}
-
-static inline void clear_sched_clock_stable(void)
-{
-}
-
-static inline void sched_clock_idle_sleep_event(void)
-{
-}
-
-static inline void sched_clock_idle_wakeup_event(u64 delta_ns)
-{
-}
-
-static inline u64 cpu_clock(int cpu)
-{
- return sched_clock();
-}
-
-static inline u64 local_clock(void)
-{
- return sched_clock();
-}
-#else
-extern void sched_clock_init_late(void);
-/*
- * Architectures can set this to 1 if they have specified
- * CONFIG_HAVE_UNSTABLE_SCHED_CLOCK in their arch Kconfig,
- * but then during bootup it turns out that sched_clock()
- * is reliable after all:
- */
-extern int sched_clock_stable(void);
-extern void clear_sched_clock_stable(void);
-
-extern void sched_clock_tick(void);
-extern void sched_clock_idle_sleep_event(void);
-extern void sched_clock_idle_wakeup_event(u64 delta_ns);
-
-/*
- * As outlined in clock.c, provides a fast, high resolution, nanosecond
- * time source that is monotonic per cpu argument and has bounded drift
- * between cpus.
- *
- * ######################### BIG FAT WARNING ##########################
- * # when comparing cpu_clock(i) to cpu_clock(j) for i != j, time can #
- * # go backwards !! #
- * ####################################################################
- */
-static inline u64 cpu_clock(int cpu)
-{
- return sched_clock_cpu(cpu);
-}
-
-static inline u64 local_clock(void)
-{
- return sched_clock_cpu(raw_smp_processor_id());
-}
-#endif
-
-#ifdef CONFIG_IRQ_TIME_ACCOUNTING
-/*
- * An i/f to runtime opt-in for irq time accounting based off of sched_clock.
- * The reason for this explicit opt-in is not to have perf penalty with
- * slow sched_clocks.
- */
-extern void enable_sched_clock_irqtime(void);
-extern void disable_sched_clock_irqtime(void);
-#else
-static inline void enable_sched_clock_irqtime(void) {}
-static inline void disable_sched_clock_irqtime(void) {}
-#endif
-
-extern unsigned long long
-task_sched_runtime(struct task_struct *task);
-
-/* sched_exec is called by processes performing an exec */
-#ifdef CONFIG_SMP
-extern void sched_exec(void);
-#else
-#define sched_exec() {}
-#endif
-
-extern void sched_clock_idle_sleep_event(void);
-extern void sched_clock_idle_wakeup_event(u64 delta_ns);
-
-#ifdef CONFIG_HOTPLUG_CPU
-extern void idle_task_exit(void);
-#else
-static inline void idle_task_exit(void) {}
-#endif
-
-#if defined(CONFIG_NO_HZ_COMMON) && defined(CONFIG_SMP)
-extern void wake_up_nohz_cpu(int cpu);
-#else
-static inline void wake_up_nohz_cpu(int cpu) { }
-#endif
-
-#ifdef CONFIG_NO_HZ_FULL
-extern u64 scheduler_tick_max_deferment(void);
-#endif
-
-#ifdef CONFIG_SCHED_AUTOGROUP
-extern void sched_autogroup_create_attach(struct task_struct *p);
-extern void sched_autogroup_detach(struct task_struct *p);
-extern void sched_autogroup_fork(struct signal_struct *sig);
-extern void sched_autogroup_exit(struct signal_struct *sig);
-extern void sched_autogroup_exit_task(struct task_struct *p);
-#ifdef CONFIG_PROC_FS
-extern void proc_sched_autogroup_show_task(struct task_struct *p, struct seq_file *m);
-extern int proc_sched_autogroup_set_nice(struct task_struct *p, int nice);
-#endif
-#else
-static inline void sched_autogroup_create_attach(struct task_struct *p) { }
-static inline void sched_autogroup_detach(struct task_struct *p) { }
-static inline void sched_autogroup_fork(struct signal_struct *sig) { }
-static inline void sched_autogroup_exit(struct signal_struct *sig) { }
-static inline void sched_autogroup_exit_task(struct task_struct *p) { }
-#endif
-
-extern int yield_to(struct task_struct *p, bool preempt);
-extern void set_user_nice(struct task_struct *p, long nice);
-extern int task_prio(const struct task_struct *p);
-/**
- * task_nice - return the nice value of a given task.
- * @p: the task in question.
- *
- * Return: The nice value [ -20 ... 0 ... 19 ].
- */
-static inline int task_nice(const struct task_struct *p)
-{
- return PRIO_TO_NICE((p)->static_prio);
-}
-extern int can_nice(const struct task_struct *p, const int nice);
-extern int task_curr(const struct task_struct *p);
-extern int idle_cpu(int cpu);
-extern int sched_setscheduler(struct task_struct *, int,
- const struct sched_param *);
-extern int sched_setscheduler_nocheck(struct task_struct *, int,
- const struct sched_param *);
-extern int sched_setattr(struct task_struct *,
- const struct sched_attr *);
-extern struct task_struct *idle_task(int cpu);
-/**
- * is_idle_task - is the specified task an idle task?
- * @p: the task in question.
- *
- * Return: 1 if @p is an idle task. 0 otherwise.
- */
-static inline bool is_idle_task(const struct task_struct *p)
-{
- return !!(p->flags & PF_IDLE);
-}
-extern struct task_struct *curr_task(int cpu);
-extern void ia64_set_curr_task(int cpu, struct task_struct *p);
-
-void yield(void);
-
-union thread_union {
-#ifndef CONFIG_THREAD_INFO_IN_TASK
- struct thread_info thread_info;
-#endif
- unsigned long stack[THREAD_SIZE/sizeof(long)];
-};
+ /*
+ * numa_faults is an array split into four regions:
+ * faults_memory, faults_cpu, faults_memory_buffer, faults_cpu_buffer
+ * in this precise order.
+ *
+ * faults_memory: Exponential decaying average of faults on a per-node
+ * basis. Scheduling placement decisions are made based on these
+ * counts. The values remain static for the duration of a PTE scan.
+ * faults_cpu: Track the nodes the process was running on when a NUMA
+ * hinting fault was incurred.
+ * faults_memory_buffer and faults_cpu_buffer: Record faults per node
+ * during the current scan window. When the scan completes, the counts
+ * in faults_memory and faults_cpu decay and these values are copied.
+ */
+ unsigned long *numa_faults;
+ unsigned long total_numa_faults;
-#ifndef __HAVE_ARCH_KSTACK_END
-static inline int kstack_end(void *addr)
-{
- /* Reliable end of stack detection:
- * Some APM bios versions misalign the stack
+ /*
+ * numa_faults_locality tracks if faults recorded during the last
+ * scan window were remote/local or failed to migrate. The task scan
+ * period is adapted based on the locality of the faults with different
+ * weights depending on whether they were shared or private faults
*/
- return !(((unsigned long)addr+sizeof(void*)-1) & (THREAD_SIZE-sizeof(void*)));
-}
-#endif
+ unsigned long numa_faults_locality[3];
-extern union thread_union init_thread_union;
-extern struct task_struct init_task;
+ unsigned long numa_pages_migrated;
+#endif /* CONFIG_NUMA_BALANCING */
-extern struct mm_struct init_mm;
+ struct tlbflush_unmap_batch tlb_ubc;
-extern struct pid_namespace init_pid_ns;
+ struct rcu_head rcu;
-/*
- * find a task by one of its numerical ids
- *
- * find_task_by_pid_ns():
- * finds a task by its pid in the specified namespace
- * find_task_by_vpid():
- * finds a task by its virtual pid
- *
- * see also find_vpid() etc in include/linux/pid.h
- */
+ /* Cache last used pipe for splice(): */
+ struct pipe_inode_info *splice_pipe;
-extern struct task_struct *find_task_by_vpid(pid_t nr);
-extern struct task_struct *find_task_by_pid_ns(pid_t nr,
- struct pid_namespace *ns);
+ struct page_frag task_frag;
-/* per-UID process charging. */
-extern struct user_struct * alloc_uid(kuid_t);
-static inline struct user_struct *get_uid(struct user_struct *u)
-{
- atomic_inc(&u->__count);
- return u;
-}
-extern void free_uid(struct user_struct *);
+#ifdef CONFIG_TASK_DELAY_ACCT
+ struct task_delay_info *delays;
+#endif
-#include <asm/current.h>
+#ifdef CONFIG_FAULT_INJECTION
+ int make_it_fail;
+#endif
+ /*
+ * When (nr_dirtied >= nr_dirtied_pause), it's time to call
+ * balance_dirty_pages() for a dirty throttling pause:
+ */
+ int nr_dirtied;
+ int nr_dirtied_pause;
+ /* Start of a write-and-pause period: */
+ unsigned long dirty_paused_when;
-extern void xtime_update(unsigned long ticks);
+#ifdef CONFIG_LATENCYTOP
+ int latency_record_count;
+ struct latency_record latency_record[LT_SAVECOUNT];
+#endif
+ /*
+ * Time slack values; these are used to round up poll() and
+ * select() etc timeout values. These are in nanoseconds.
+ */
+ u64 timer_slack_ns;
+ u64 default_timer_slack_ns;
-extern int wake_up_state(struct task_struct *tsk, unsigned int state);
-extern int wake_up_process(struct task_struct *tsk);
-extern void wake_up_new_task(struct task_struct *tsk);
-#ifdef CONFIG_SMP
- extern void kick_process(struct task_struct *tsk);
-#else
- static inline void kick_process(struct task_struct *tsk) { }
+#ifdef CONFIG_KASAN
+ unsigned int kasan_depth;
#endif
-extern int sched_fork(unsigned long clone_flags, struct task_struct *p);
-extern void sched_dead(struct task_struct *p);
-extern void proc_caches_init(void);
-extern void flush_signals(struct task_struct *);
-extern void ignore_signals(struct task_struct *);
-extern void flush_signal_handlers(struct task_struct *, int force_default);
-extern int dequeue_signal(struct task_struct *tsk, sigset_t *mask, siginfo_t *info);
+#ifdef CONFIG_FUNCTION_GRAPH_TRACER
+ /* Index of current stored address in ret_stack: */
+ int curr_ret_stack;
-static inline int kernel_dequeue_signal(siginfo_t *info)
-{
- struct task_struct *tsk = current;
- siginfo_t __info;
- int ret;
+ /* Stack of return addresses for return function tracing: */
+ struct ftrace_ret_stack *ret_stack;
- spin_lock_irq(&tsk->sighand->siglock);
- ret = dequeue_signal(tsk, &tsk->blocked, info ?: &__info);
- spin_unlock_irq(&tsk->sighand->siglock);
+ /* Timestamp for last schedule: */
+ unsigned long long ftrace_timestamp;
- return ret;
-}
+ /*
+ * Number of functions that haven't been traced
+ * because of depth overrun:
+ */
+ atomic_t trace_overrun;
-static inline void kernel_signal_stop(void)
-{
- spin_lock_irq(¤t->sighand->siglock);
- if (current->jobctl & JOBCTL_STOP_DEQUEUED)
- __set_current_state(TASK_STOPPED);
- spin_unlock_irq(¤t->sighand->siglock);
+ /* Pause tracing: */
+ atomic_t tracing_graph_pause;
+#endif
- schedule();
-}
+#ifdef CONFIG_TRACING
+ /* State flags for use by tracers: */
+ unsigned long trace;
-extern void release_task(struct task_struct * p);
-extern int send_sig_info(int, struct siginfo *, struct task_struct *);
-extern int force_sigsegv(int, struct task_struct *);
-extern int force_sig_info(int, struct siginfo *, struct task_struct *);
-extern int __kill_pgrp_info(int sig, struct siginfo *info, struct pid *pgrp);
-extern int kill_pid_info(int sig, struct siginfo *info, struct pid *pid);
-extern int kill_pid_info_as_cred(int, struct siginfo *, struct pid *,
- const struct cred *, u32);
-extern int kill_pgrp(struct pid *pid, int sig, int priv);
-extern int kill_pid(struct pid *pid, int sig, int priv);
-extern int kill_proc_info(int, struct siginfo *, pid_t);
-extern __must_check bool do_notify_parent(struct task_struct *, int);
-extern void __wake_up_parent(struct task_struct *p, struct task_struct *parent);
-extern void force_sig(int, struct task_struct *);
-extern int send_sig(int, struct task_struct *, int);
-extern int zap_other_threads(struct task_struct *p);
-extern struct sigqueue *sigqueue_alloc(void);
-extern void sigqueue_free(struct sigqueue *);
-extern int send_sigqueue(struct sigqueue *, struct task_struct *, int group);
-extern int do_sigaction(int, struct k_sigaction *, struct k_sigaction *);
-
-#ifdef TIF_RESTORE_SIGMASK
-/*
- * Legacy restore_sigmask accessors. These are inefficient on
- * SMP architectures because they require atomic operations.
- */
+ /* Bitmask and counter of trace recursion: */
+ unsigned long trace_recursion;
+#endif /* CONFIG_TRACING */
-/**
- * set_restore_sigmask() - make sure saved_sigmask processing gets done
- *
- * This sets TIF_RESTORE_SIGMASK and ensures that the arch signal code
- * will run before returning to user mode, to process the flag. For
- * all callers, TIF_SIGPENDING is already set or it's no harm to set
- * it. TIF_RESTORE_SIGMASK need not be in the set of bits that the
- * arch code will notice on return to user mode, in case those bits
- * are scarce. We set TIF_SIGPENDING here to ensure that the arch
- * signal code always gets run when TIF_RESTORE_SIGMASK is set.
- */
-static inline void set_restore_sigmask(void)
-{
- set_thread_flag(TIF_RESTORE_SIGMASK);
- WARN_ON(!test_thread_flag(TIF_SIGPENDING));
-}
-static inline void clear_restore_sigmask(void)
-{
- clear_thread_flag(TIF_RESTORE_SIGMASK);
-}
-static inline bool test_restore_sigmask(void)
-{
- return test_thread_flag(TIF_RESTORE_SIGMASK);
-}
-static inline bool test_and_clear_restore_sigmask(void)
-{
- return test_and_clear_thread_flag(TIF_RESTORE_SIGMASK);
-}
+#ifdef CONFIG_KCOV
+ /* Coverage collection mode enabled for this task (0 if disabled): */
+ enum kcov_mode kcov_mode;
-#else /* TIF_RESTORE_SIGMASK */
+ /* Size of the kcov_area: */
+ unsigned int kcov_size;
-/* Higher-quality implementation, used if TIF_RESTORE_SIGMASK doesn't exist. */
-static inline void set_restore_sigmask(void)
-{
- current->restore_sigmask = true;
- WARN_ON(!test_thread_flag(TIF_SIGPENDING));
-}
-static inline void clear_restore_sigmask(void)
-{
- current->restore_sigmask = false;
-}
-static inline bool test_restore_sigmask(void)
-{
- return current->restore_sigmask;
-}
-static inline bool test_and_clear_restore_sigmask(void)
-{
- if (!current->restore_sigmask)
- return false;
- current->restore_sigmask = false;
- return true;
-}
-#endif
+ /* Buffer for coverage collection: */
+ void *kcov_area;
-static inline void restore_saved_sigmask(void)
-{
- if (test_and_clear_restore_sigmask())
- __set_current_blocked(¤t->saved_sigmask);
-}
+ /* KCOV descriptor wired with this task or NULL: */
+ struct kcov *kcov;
+#endif
-static inline sigset_t *sigmask_to_save(void)
-{
- sigset_t *res = ¤t->blocked;
- if (unlikely(test_restore_sigmask()))
- res = ¤t->saved_sigmask;
- return res;
-}
+#ifdef CONFIG_MEMCG
+ struct mem_cgroup *memcg_in_oom;
+ gfp_t memcg_oom_gfp_mask;
+ int memcg_oom_order;
-static inline int kill_cad_pid(int sig, int priv)
-{
- return kill_pid(cad_pid, sig, priv);
-}
+ /* Number of pages to reclaim on returning to userland: */
+ unsigned int memcg_nr_pages_over_high;
+#endif
-/* These can be the second arg to send_sig_info/send_group_sig_info. */
-#define SEND_SIG_NOINFO ((struct siginfo *) 0)
-#define SEND_SIG_PRIV ((struct siginfo *) 1)
-#define SEND_SIG_FORCED ((struct siginfo *) 2)
+#ifdef CONFIG_UPROBES
+ struct uprobe_task *utask;
+#endif
+#if defined(CONFIG_BCACHE) || defined(CONFIG_BCACHE_MODULE)
+ unsigned int sequential_io;
+ unsigned int sequential_io_avg;
+#endif
+#ifdef CONFIG_DEBUG_ATOMIC_SLEEP
+ unsigned long task_state_change;
+#endif
+ int pagefault_disabled;
+#ifdef CONFIG_MMU
+ struct task_struct *oom_reaper_list;
+#endif
+#ifdef CONFIG_VMAP_STACK
+ struct vm_struct *stack_vm_area;
+#endif
+#ifdef CONFIG_THREAD_INFO_IN_TASK
+ /* A live task holds one reference: */
+ atomic_t stack_refcount;
+#endif
+ /* CPU-specific state of this task: */
+ struct thread_struct thread;
-/*
- * True if we are on the alternate signal stack.
- */
-static inline int on_sig_stack(unsigned long sp)
-{
/*
- * If the signal stack is SS_AUTODISARM then, by construction, we
- * can't be on the signal stack unless user code deliberately set
- * SS_AUTODISARM when we were already on it.
+ * WARNING: on x86, 'thread_struct' contains a variable-sized
+ * structure. It *MUST* be at the end of 'task_struct'.
*
- * This improves reliability: if user state gets corrupted such that
- * the stack pointer points very close to the end of the signal stack,
- * then this check will enable the signal to be handled anyway.
+ * Do not put anything below here!
*/
- if (current->sas_ss_flags & SS_AUTODISARM)
- return 0;
+};
-#ifdef CONFIG_STACK_GROWSUP
- return sp >= current->sas_ss_sp &&
- sp - current->sas_ss_sp < current->sas_ss_size;
-#else
- return sp > current->sas_ss_sp &&
- sp - current->sas_ss_sp <= current->sas_ss_size;
-#endif
+static inline struct pid *task_pid(struct task_struct *task)
+{
+ return task->pids[PIDTYPE_PID].pid;
}
-static inline int sas_ss_flags(unsigned long sp)
+static inline struct pid *task_tgid(struct task_struct *task)
{
- if (!current->sas_ss_size)
- return SS_DISABLE;
-
- return on_sig_stack(sp) ? SS_ONSTACK : 0;
+ return task->group_leader->pids[PIDTYPE_PID].pid;
}
-static inline void sas_ss_reset(struct task_struct *p)
+/*
+ * Without tasklist or RCU lock it is not safe to dereference
+ * the result of task_pgrp/task_session even if task == current,
+ * we can race with another thread doing sys_setsid/sys_setpgid.
+ */
+static inline struct pid *task_pgrp(struct task_struct *task)
{
- p->sas_ss_sp = 0;
- p->sas_ss_size = 0;
- p->sas_ss_flags = SS_DISABLE;
+ return task->group_leader->pids[PIDTYPE_PGID].pid;
}
-static inline unsigned long sigsp(unsigned long sp, struct ksignal *ksig)
+static inline struct pid *task_session(struct task_struct *task)
{
- if (unlikely((ksig->ka.sa.sa_flags & SA_ONSTACK)) && ! sas_ss_flags(sp))
-#ifdef CONFIG_STACK_GROWSUP
- return current->sas_ss_sp;
-#else
- return current->sas_ss_sp + current->sas_ss_size;
-#endif
- return sp;
+ return task->group_leader->pids[PIDTYPE_SID].pid;
}
/*
- * Routines for handling mm_structs
- */
-extern struct mm_struct * mm_alloc(void);
-
-/**
- * mmgrab() - Pin a &struct mm_struct.
- * @mm: The &struct mm_struct to pin.
- *
- * Make sure that @mm will not get freed even after the owning task
- * exits. This doesn't guarantee that the associated address space
- * will still exist later on and mmget_not_zero() has to be used before
- * accessing it.
+ * the helpers to get the task's different pids as they are seen
+ * from various namespaces
*
- * This is a preferred way to to pin @mm for a longer/unbounded amount
- * of time.
+ * task_xid_nr() : global id, i.e. the id seen from the init namespace;
+ * task_xid_vnr() : virtual id, i.e. the id seen from the pid namespace of
+ * current.
+ * task_xid_nr_ns() : id seen from the ns specified;
*
- * Use mmdrop() to release the reference acquired by mmgrab().
+ * set_task_vxid() : assigns a virtual id to a task;
*
- * See also <Documentation/vm/active_mm.txt> for an in-depth explanation
- * of &mm_struct.mm_count vs &mm_struct.mm_users.
+ * see also pid_nr() etc in include/linux/pid.h
*/
-static inline void mmgrab(struct mm_struct *mm)
+pid_t __task_pid_nr_ns(struct task_struct *task, enum pid_type type, struct pid_namespace *ns);
+
+static inline pid_t task_pid_nr(struct task_struct *tsk)
+{
+ return tsk->pid;
+}
+
+static inline pid_t task_pid_nr_ns(struct task_struct *tsk, struct pid_namespace *ns)
{
- atomic_inc(&mm->mm_count);
+ return __task_pid_nr_ns(tsk, PIDTYPE_PID, ns);
}
-/* mmdrop drops the mm and the page tables */
-extern void __mmdrop(struct mm_struct *);
-static inline void mmdrop(struct mm_struct *mm)
+static inline pid_t task_pid_vnr(struct task_struct *tsk)
{
- if (unlikely(atomic_dec_and_test(&mm->mm_count)))
- __mmdrop(mm);
+ return __task_pid_nr_ns(tsk, PIDTYPE_PID, NULL);
}
-static inline void mmdrop_async_fn(struct work_struct *work)
+
+static inline pid_t task_tgid_nr(struct task_struct *tsk)
{
- struct mm_struct *mm = container_of(work, struct mm_struct, async_put_work);
- __mmdrop(mm);
+ return tsk->tgid;
}
-static inline void mmdrop_async(struct mm_struct *mm)
+extern pid_t task_tgid_nr_ns(struct task_struct *tsk, struct pid_namespace *ns);
+
+static inline pid_t task_tgid_vnr(struct task_struct *tsk)
{
- if (unlikely(atomic_dec_and_test(&mm->mm_count))) {
- INIT_WORK(&mm->async_put_work, mmdrop_async_fn);
- schedule_work(&mm->async_put_work);
- }
+ return pid_vnr(task_tgid(tsk));
}
/**
- * mmget() - Pin the address space associated with a &struct mm_struct.
- * @mm: The address space to pin.
- *
- * Make sure that the address space of the given &struct mm_struct doesn't
- * go away. This does not protect against parts of the address space being
- * modified or freed, however.
- *
- * Never use this function to pin this address space for an
- * unbounded/indefinite amount of time.
+ * pid_alive - check that a task structure is not stale
+ * @p: Task structure to be checked.
*
- * Use mmput() to release the reference acquired by mmget().
+ * Test if a process is not yet dead (at most zombie state)
+ * If pid_alive fails, then pointers within the task structure
+ * can be stale and must not be dereferenced.
*
- * See also <Documentation/vm/active_mm.txt> for an in-depth explanation
- * of &mm_struct.mm_count vs &mm_struct.mm_users.
+ * Return: 1 if the process is alive. 0 otherwise.
*/
-static inline void mmget(struct mm_struct *mm)
+static inline int pid_alive(const struct task_struct *p)
{
- atomic_inc(&mm->mm_users);
+ return p->pids[PIDTYPE_PID].pid != NULL;
}
-static inline bool mmget_not_zero(struct mm_struct *mm)
+static inline pid_t task_ppid_nr_ns(const struct task_struct *tsk, struct pid_namespace *ns)
{
- return atomic_inc_not_zero(&mm->mm_users);
-}
+ pid_t pid = 0;
-/* mmput gets rid of the mappings and all user-space */
-extern void mmput(struct mm_struct *);
-#ifdef CONFIG_MMU
-/* same as above but performs the slow path from the async context. Can
- * be called from the atomic context as well
- */
-extern void mmput_async(struct mm_struct *);
-#endif
+ rcu_read_lock();
+ if (pid_alive(tsk))
+ pid = task_tgid_nr_ns(rcu_dereference(tsk->real_parent), ns);
+ rcu_read_unlock();
-/* Grab a reference to a task's mm, if it is not already going away */
-extern struct mm_struct *get_task_mm(struct task_struct *task);
-/*
- * Grab a reference to a task's mm, if it is not already going away
- * and ptrace_may_access with the mode parameter passed to it
- * succeeds.
- */
-extern struct mm_struct *mm_access(struct task_struct *task, unsigned int mode);
-/* Remove the current tasks stale references to the old mm_struct */
-extern void mm_release(struct task_struct *, struct mm_struct *);
+ return pid;
+}
-#ifdef CONFIG_HAVE_COPY_THREAD_TLS
-extern int copy_thread_tls(unsigned long, unsigned long, unsigned long,
- struct task_struct *, unsigned long);
-#else
-extern int copy_thread(unsigned long, unsigned long, unsigned long,
- struct task_struct *);
-
-/* Architectures that haven't opted into copy_thread_tls get the tls argument
- * via pt_regs, so ignore the tls argument passed via C. */
-static inline int copy_thread_tls(
- unsigned long clone_flags, unsigned long sp, unsigned long arg,
- struct task_struct *p, unsigned long tls)
+static inline pid_t task_ppid_nr(const struct task_struct *tsk)
{
- return copy_thread(clone_flags, sp, arg, p);
+ return task_ppid_nr_ns(tsk, &init_pid_ns);
}
-#endif
-extern void flush_thread(void);
-#ifdef CONFIG_HAVE_EXIT_THREAD
-extern void exit_thread(struct task_struct *tsk);
-#else
-static inline void exit_thread(struct task_struct *tsk)
+static inline pid_t task_pgrp_nr_ns(struct task_struct *tsk, struct pid_namespace *ns)
{
+ return __task_pid_nr_ns(tsk, PIDTYPE_PGID, ns);
}
-#endif
-
-extern void exit_files(struct task_struct *);
-extern void __cleanup_sighand(struct sighand_struct *);
-extern void exit_itimers(struct signal_struct *);
-extern void flush_itimer_signals(void);
-
-extern void do_group_exit(int);
+static inline pid_t task_pgrp_vnr(struct task_struct *tsk)
+{
+ return __task_pid_nr_ns(tsk, PIDTYPE_PGID, NULL);
+}
-extern int do_execve(struct filename *,
- const char __user * const __user *,
- const char __user * const __user *);
-extern int do_execveat(int, struct filename *,
- const char __user * const __user *,
- const char __user * const __user *,
- int);
-extern long _do_fork(unsigned long, unsigned long, unsigned long, int __user *, int __user *, unsigned long);
-extern long do_fork(unsigned long, unsigned long, unsigned long, int __user *, int __user *);
-struct task_struct *fork_idle(int);
-extern pid_t kernel_thread(int (*fn)(void *), void *arg, unsigned long flags);
-extern void __set_task_comm(struct task_struct *tsk, const char *from, bool exec);
-static inline void set_task_comm(struct task_struct *tsk, const char *from)
+static inline pid_t task_session_nr_ns(struct task_struct *tsk, struct pid_namespace *ns)
{
- __set_task_comm(tsk, from, false);
+ return __task_pid_nr_ns(tsk, PIDTYPE_SID, ns);
}
-extern char *get_task_comm(char *to, struct task_struct *tsk);
-#ifdef CONFIG_SMP
-void scheduler_ipi(void);
-extern unsigned long wait_task_inactive(struct task_struct *, long match_state);
-#else
-static inline void scheduler_ipi(void) { }
-static inline unsigned long wait_task_inactive(struct task_struct *p,
- long match_state)
+static inline pid_t task_session_vnr(struct task_struct *tsk)
{
- return 1;
+ return __task_pid_nr_ns(tsk, PIDTYPE_SID, NULL);
}
-#endif
-#define tasklist_empty() \
- list_empty(&init_task.tasks)
+/* Obsolete, do not use: */
+static inline pid_t task_pgrp_nr(struct task_struct *tsk)
+{
+ return task_pgrp_nr_ns(tsk, &init_pid_ns);
+}
-#define next_task(p) \
- list_entry_rcu((p)->tasks.next, struct task_struct, tasks)
+/**
+ * is_global_init - check if a task structure is init. Since init
+ * is free to have sub-threads we need to check tgid.
+ * @tsk: Task structure to be checked.
+ *
+ * Check if a task structure is the first user space task the kernel created.
+ *
+ * Return: 1 if the task structure is init. 0 otherwise.
+ */
+static inline int is_global_init(struct task_struct *tsk)
+{
+ return task_tgid_nr(tsk) == 1;
+}
-#define for_each_process(p) \
- for (p = &init_task ; (p = next_task(p)) != &init_task ; )
+extern struct pid *cad_pid;
-extern bool current_is_single_threaded(void);
+/*
+ * Per process flags
+ */
+#define PF_IDLE 0x00000002 /* I am an IDLE thread */
+#define PF_EXITING 0x00000004 /* Getting shut down */
+#define PF_EXITPIDONE 0x00000008 /* PI exit done on shut down */
+#define PF_VCPU 0x00000010 /* I'm a virtual CPU */
+#define PF_WQ_WORKER 0x00000020 /* I'm a workqueue worker */
+#define PF_FORKNOEXEC 0x00000040 /* Forked but didn't exec */
+#define PF_MCE_PROCESS 0x00000080 /* Process policy on mce errors */
+#define PF_SUPERPRIV 0x00000100 /* Used super-user privileges */
+#define PF_DUMPCORE 0x00000200 /* Dumped core */
+#define PF_SIGNALED 0x00000400 /* Killed by a signal */
+#define PF_MEMALLOC 0x00000800 /* Allocating memory */
+#define PF_NPROC_EXCEEDED 0x00001000 /* set_user() noticed that RLIMIT_NPROC was exceeded */
+#define PF_USED_MATH 0x00002000 /* If unset the fpu must be initialized before use */
+#define PF_USED_ASYNC 0x00004000 /* Used async_schedule*(), used by module init */
+#define PF_NOFREEZE 0x00008000 /* This thread should not be frozen */
+#define PF_FROZEN 0x00010000 /* Frozen for system suspend */
+#define PF_FSTRANS 0x00020000 /* Inside a filesystem transaction */
+#define PF_KSWAPD 0x00040000 /* I am kswapd */
+#define PF_MEMALLOC_NOIO 0x00080000 /* Allocating memory without IO involved */
+#define PF_LESS_THROTTLE 0x00100000 /* Throttle me less: I clean memory */
+#define PF_KTHREAD 0x00200000 /* I am a kernel thread */
+#define PF_RANDOMIZE 0x00400000 /* Randomize virtual address space */
+#define PF_SWAPWRITE 0x00800000 /* Allowed to write to swap */
+#define PF_NO_SETAFFINITY 0x04000000 /* Userland is not allowed to meddle with cpus_allowed */
+#define PF_MCE_EARLY 0x08000000 /* Early kill for mce process policy */
+#define PF_MUTEX_TESTER 0x20000000 /* Thread belongs to the rt mutex tester */
+#define PF_FREEZER_SKIP 0x40000000 /* Freezer should not count it as freezable */
+#define PF_SUSPEND_TASK 0x80000000 /* This thread called freeze_processes() and should not be frozen */
/*
- * Careful: do_each_thread/while_each_thread is a double loop so
- * 'break' will not work as expected - use goto instead.
+ * Only the _current_ task can read/write to tsk->flags, but other
+ * tasks can access tsk->flags in readonly mode for example
+ * with tsk_used_math (like during threaded core dumping).
+ * There is however an exception to this rule during ptrace
+ * or during fork: the ptracer task is allowed to write to the
+ * child->flags of its traced child (same goes for fork, the parent
+ * can write to the child->flags), because we're guaranteed the
+ * child is not running and in turn not changing child->flags
+ * at the same time the parent does it.
*/
-#define do_each_thread(g, t) \
- for (g = t = &init_task ; (g = t = next_task(g)) != &init_task ; ) do
+#define clear_stopped_child_used_math(child) do { (child)->flags &= ~PF_USED_MATH; } while (0)
+#define set_stopped_child_used_math(child) do { (child)->flags |= PF_USED_MATH; } while (0)
+#define clear_used_math() clear_stopped_child_used_math(current)
+#define set_used_math() set_stopped_child_used_math(current)
+
+#define conditional_stopped_child_used_math(condition, child) \
+ do { (child)->flags &= ~PF_USED_MATH, (child)->flags |= (condition) ? PF_USED_MATH : 0; } while (0)
-#define while_each_thread(g, t) \
- while ((t = next_thread(t)) != g)
+#define conditional_used_math(condition) conditional_stopped_child_used_math(condition, current)
-#define __for_each_thread(signal, t) \
- list_for_each_entry_rcu(t, &(signal)->thread_head, thread_node)
+#define copy_to_stopped_child_used_math(child) \
+ do { (child)->flags &= ~PF_USED_MATH, (child)->flags |= current->flags & PF_USED_MATH; } while (0)
-#define for_each_thread(p, t) \
- __for_each_thread((p)->signal, t)
+/* NOTE: this will return 0 or PF_USED_MATH, it will never return 1 */
+#define tsk_used_math(p) ((p)->flags & PF_USED_MATH)
+#define used_math() tsk_used_math(current)
-/* Careful: this is a double loop, 'break' won't work as expected. */
-#define for_each_process_thread(p, t) \
- for_each_process(p) for_each_thread(p, t)
+/* Per-process atomic flags. */
+#define PFA_NO_NEW_PRIVS 0 /* May not gain new privileges. */
+#define PFA_SPREAD_PAGE 1 /* Spread page cache over cpuset */
+#define PFA_SPREAD_SLAB 2 /* Spread some slab caches over cpuset */
+#define PFA_LMK_WAITING 3 /* Lowmemorykiller is waiting */
-typedef int (*proc_visitor)(struct task_struct *p, void *data);
-void walk_process_tree(struct task_struct *top, proc_visitor, void *);
-static inline int get_nr_threads(struct task_struct *tsk)
-{
- return tsk->signal->nr_threads;
-}
+#define TASK_PFA_TEST(name, func) \
+ static inline bool task_##func(struct task_struct *p) \
+ { return test_bit(PFA_##name, &p->atomic_flags); }
-static inline bool thread_group_leader(struct task_struct *p)
-{
- return p->exit_signal >= 0;
-}
+#define TASK_PFA_SET(name, func) \
+ static inline void task_set_##func(struct task_struct *p) \
+ { set_bit(PFA_##name, &p->atomic_flags); }
-/* Do to the insanities of de_thread it is possible for a process
- * to have the pid of the thread group leader without actually being
- * the thread group leader. For iteration through the pids in proc
- * all we care about is that we have a task with the appropriate
- * pid, we don't actually care if we have the right task.
- */
-static inline bool has_group_leader_pid(struct task_struct *p)
-{
- return task_pid(p) == p->signal->leader_pid;
-}
+#define TASK_PFA_CLEAR(name, func) \
+ static inline void task_clear_##func(struct task_struct *p) \
+ { clear_bit(PFA_##name, &p->atomic_flags); }
-static inline
-bool same_thread_group(struct task_struct *p1, struct task_struct *p2)
-{
- return p1->signal == p2->signal;
-}
+TASK_PFA_TEST(NO_NEW_PRIVS, no_new_privs)
+TASK_PFA_SET(NO_NEW_PRIVS, no_new_privs)
-static inline struct task_struct *next_thread(const struct task_struct *p)
-{
- return list_entry_rcu(p->thread_group.next,
- struct task_struct, thread_group);
-}
+TASK_PFA_TEST(SPREAD_PAGE, spread_page)
+TASK_PFA_SET(SPREAD_PAGE, spread_page)
+TASK_PFA_CLEAR(SPREAD_PAGE, spread_page)
-static inline int thread_group_empty(struct task_struct *p)
-{
- return list_empty(&p->thread_group);
-}
+TASK_PFA_TEST(SPREAD_SLAB, spread_slab)
+TASK_PFA_SET(SPREAD_SLAB, spread_slab)
+TASK_PFA_CLEAR(SPREAD_SLAB, spread_slab)
-#define delay_group_leader(p) \
- (thread_group_leader(p) && !thread_group_empty(p))
+TASK_PFA_TEST(LMK_WAITING, lmk_waiting)
+TASK_PFA_SET(LMK_WAITING, lmk_waiting)
-/*
- * Protects ->fs, ->files, ->mm, ->group_info, ->comm, keyring
- * subscriptions and synchronises with wait4(). Also used in procfs. Also
- * pins the final release of task.io_context. Also protects ->cpuset and
- * ->cgroup.subsys[]. And ->vfork_done.
- *
- * Nests both inside and outside of read_lock(&tasklist_lock).
- * It must not be nested with write_lock_irq(&tasklist_lock),
- * neither inside nor outside.
- */
-static inline void task_lock(struct task_struct *p)
+static inline void
+tsk_restore_flags(struct task_struct *task, unsigned long orig_flags, unsigned long flags)
{
- spin_lock(&p->alloc_lock);
+ task->flags &= ~flags;
+ task->flags |= orig_flags & flags;
}
-static inline void task_unlock(struct task_struct *p)
+extern int cpuset_cpumask_can_shrink(const struct cpumask *cur, const struct cpumask *trial);
+extern int task_can_attach(struct task_struct *p, const struct cpumask *cs_cpus_allowed);
+#ifdef CONFIG_SMP
+extern void do_set_cpus_allowed(struct task_struct *p, const struct cpumask *new_mask);
+extern int set_cpus_allowed_ptr(struct task_struct *p, const struct cpumask *new_mask);
+#else
+static inline void do_set_cpus_allowed(struct task_struct *p, const struct cpumask *new_mask)
{
- spin_unlock(&p->alloc_lock);
}
-
-extern struct sighand_struct *__lock_task_sighand(struct task_struct *tsk,
- unsigned long *flags);
-
-static inline struct sighand_struct *lock_task_sighand(struct task_struct *tsk,
- unsigned long *flags)
+static inline int set_cpus_allowed_ptr(struct task_struct *p, const struct cpumask *new_mask)
{
- struct sighand_struct *ret;
-
- ret = __lock_task_sighand(tsk, flags);
- (void)__cond_lock(&tsk->sighand->siglock, ret);
- return ret;
+ if (!cpumask_test_cpu(0, new_mask))
+ return -EINVAL;
+ return 0;
}
+#endif
-static inline void unlock_task_sighand(struct task_struct *tsk,
- unsigned long *flags)
-{
- spin_unlock_irqrestore(&tsk->sighand->siglock, *flags);
-}
+#ifndef cpu_relax_yield
+#define cpu_relax_yield() cpu_relax()
+#endif
+
+extern int yield_to(struct task_struct *p, bool preempt);
+extern void set_user_nice(struct task_struct *p, long nice);
+extern int task_prio(const struct task_struct *p);
/**
- * threadgroup_change_begin - mark the beginning of changes to a threadgroup
- * @tsk: task causing the changes
+ * task_nice - return the nice value of a given task.
+ * @p: the task in question.
*
- * All operations which modify a threadgroup - a new thread joining the
- * group, death of a member thread (the assertion of PF_EXITING) and
- * exec(2) dethreading the process and replacing the leader - are wrapped
- * by threadgroup_change_{begin|end}(). This is to provide a place which
- * subsystems needing threadgroup stability can hook into for
- * synchronization.
+ * Return: The nice value [ -20 ... 0 ... 19 ].
*/
-static inline void threadgroup_change_begin(struct task_struct *tsk)
+static inline int task_nice(const struct task_struct *p)
{
- might_sleep();
- cgroup_threadgroup_change_begin(tsk);
+ return PRIO_TO_NICE((p)->static_prio);
}
+extern int can_nice(const struct task_struct *p, const int nice);
+extern int task_curr(const struct task_struct *p);
+extern int idle_cpu(int cpu);
+extern int sched_setscheduler(struct task_struct *, int, const struct sched_param *);
+extern int sched_setscheduler_nocheck(struct task_struct *, int, const struct sched_param *);
+extern int sched_setattr(struct task_struct *, const struct sched_attr *);
+extern struct task_struct *idle_task(int cpu);
+
/**
- * threadgroup_change_end - mark the end of changes to a threadgroup
- * @tsk: task causing the changes
+ * is_idle_task - is the specified task an idle task?
+ * @p: the task in question.
*
- * See threadgroup_change_begin().
+ * Return: 1 if @p is an idle task. 0 otherwise.
*/
-static inline void threadgroup_change_end(struct task_struct *tsk)
+static inline bool is_idle_task(const struct task_struct *p)
{
- cgroup_threadgroup_change_end(tsk);
+ return !!(p->flags & PF_IDLE);
}
-#ifdef CONFIG_THREAD_INFO_IN_TASK
-
-static inline struct thread_info *task_thread_info(struct task_struct *task)
-{
- return &task->thread_info;
-}
+extern struct task_struct *curr_task(int cpu);
+extern void ia64_set_curr_task(int cpu, struct task_struct *p);
-/*
- * When accessing the stack of a non-current task that might exit, use
- * try_get_task_stack() instead. task_stack_page will return a pointer
- * that could get freed out from under you.
- */
-static inline void *task_stack_page(const struct task_struct *task)
-{
- return task->stack;
-}
+void yield(void);
-#define setup_thread_stack(new,old) do { } while(0)
+union thread_union {
+#ifndef CONFIG_THREAD_INFO_IN_TASK
+ struct thread_info thread_info;
+#endif
+ unsigned long stack[THREAD_SIZE/sizeof(long)];
+};
-static inline unsigned long *end_of_stack(const struct task_struct *task)
+#ifdef CONFIG_THREAD_INFO_IN_TASK
+static inline struct thread_info *task_thread_info(struct task_struct *task)
{
- return task->stack;
+ return &task->thread_info;
}
-
#elif !defined(__HAVE_THREAD_FUNCTIONS)
-
-#define task_thread_info(task) ((struct thread_info *)(task)->stack)
-#define task_stack_page(task) ((void *)(task)->stack)
-
-static inline void setup_thread_stack(struct task_struct *p, struct task_struct *org)
-{
- *task_thread_info(p) = *task_thread_info(org);
- task_thread_info(p)->task = p;
-}
+# define task_thread_info(task) ((struct thread_info *)(task)->stack)
+#endif
/*
- * Return the address of the last usable long on the stack.
+ * find a task by one of its numerical ids
*
- * When the stack grows down, this is just above the thread
- * info struct. Going any lower will corrupt the threadinfo.
+ * find_task_by_pid_ns():
+ * finds a task by its pid in the specified namespace
+ * find_task_by_vpid():
+ * finds a task by its virtual pid
*
- * When the stack grows up, this is the highest address.
- * Beyond that position, we corrupt data on the next page.
+ * see also find_vpid() etc in include/linux/pid.h
*/
-static inline unsigned long *end_of_stack(struct task_struct *p)
-{
-#ifdef CONFIG_STACK_GROWSUP
- return (unsigned long *)((unsigned long)task_thread_info(p) + THREAD_SIZE) - 1;
-#else
- return (unsigned long *)(task_thread_info(p) + 1);
-#endif
-}
-#endif
+extern struct task_struct *find_task_by_vpid(pid_t nr);
+extern struct task_struct *find_task_by_pid_ns(pid_t nr, struct pid_namespace *ns);
-#ifdef CONFIG_THREAD_INFO_IN_TASK
-static inline void *try_get_task_stack(struct task_struct *tsk)
-{
- return atomic_inc_not_zero(&tsk->stack_refcount) ?
- task_stack_page(tsk) : NULL;
-}
+extern int wake_up_state(struct task_struct *tsk, unsigned int state);
+extern int wake_up_process(struct task_struct *tsk);
+extern void wake_up_new_task(struct task_struct *tsk);
-extern void put_task_stack(struct task_struct *tsk);
+#ifdef CONFIG_SMP
+extern void kick_process(struct task_struct *tsk);
#else
-static inline void *try_get_task_stack(struct task_struct *tsk)
-{
- return task_stack_page(tsk);
-}
-
-static inline void put_task_stack(struct task_struct *tsk) {}
+static inline void kick_process(struct task_struct *tsk) { }
#endif
-#define task_stack_end_corrupted(task) \
- (*(end_of_stack(task)) != STACK_END_MAGIC)
+extern void __set_task_comm(struct task_struct *tsk, const char *from, bool exec);
-static inline int object_is_on_stack(void *obj)
+static inline void set_task_comm(struct task_struct *tsk, const char *from)
{
- void *stack = task_stack_page(current);
-
- return (obj >= stack) && (obj < (stack + THREAD_SIZE));
+ __set_task_comm(tsk, from, false);
}
-extern void thread_stack_cache_init(void);
+extern char *get_task_comm(char *to, struct task_struct *tsk);
-#ifdef CONFIG_DEBUG_STACK_USAGE
-static inline unsigned long stack_not_used(struct task_struct *p)
+#ifdef CONFIG_SMP
+void scheduler_ipi(void);
+extern unsigned long wait_task_inactive(struct task_struct *, long match_state);
+#else
+static inline void scheduler_ipi(void) { }
+static inline unsigned long wait_task_inactive(struct task_struct *p, long match_state)
{
- unsigned long *n = end_of_stack(p);
-
- do { /* Skip over canary */
-# ifdef CONFIG_STACK_GROWSUP
- n--;
-# else
- n++;
-# endif
- } while (!*n);
-
-# ifdef CONFIG_STACK_GROWSUP
- return (unsigned long)end_of_stack(p) - (unsigned long)n;
-# else
- return (unsigned long)n - (unsigned long)end_of_stack(p);
-# endif
+ return 1;
}
#endif
-extern void set_task_stack_end_magic(struct task_struct *tsk);
-/* set thread flags in other task's structures
- * - see asm/thread_info.h for TIF_xxxx flags available
+/*
+ * Set thread flags in other task's structures.
+ * See asm/thread_info.h for TIF_xxxx flags available:
*/
static inline void set_tsk_thread_flag(struct task_struct *tsk, int flag)
{
return unlikely(test_tsk_thread_flag(tsk,TIF_NEED_RESCHED));
}
-static inline int restart_syscall(void)
-{
- set_tsk_thread_flag(current, TIF_SIGPENDING);
- return -ERESTARTNOINTR;
-}
-
-static inline int signal_pending(struct task_struct *p)
-{
- return unlikely(test_tsk_thread_flag(p,TIF_SIGPENDING));
-}
-
-static inline int __fatal_signal_pending(struct task_struct *p)
-{
- return unlikely(sigismember(&p->pending.signal, SIGKILL));
-}
-
-static inline int fatal_signal_pending(struct task_struct *p)
-{
- return signal_pending(p) && __fatal_signal_pending(p);
-}
-
-static inline int signal_pending_state(long state, struct task_struct *p)
-{
- if (!(state & (TASK_INTERRUPTIBLE | TASK_WAKEKILL)))
- return 0;
- if (!signal_pending(p))
- return 0;
-
- return (state & TASK_INTERRUPTIBLE) || __fatal_signal_pending(p);
-}
-
/*
* cond_resched() and cond_resched_lock(): latency reduction via
* explicit rescheduling in places that are safe. The return
#endif
}
-static inline unsigned long get_preempt_disable_ip(struct task_struct *p)
-{
-#ifdef CONFIG_DEBUG_PREEMPT
- return p->preempt_disable_ip;
-#else
- return 0;
-#endif
-}
-
/*
* Does a critical section need to be broken due to another
* task waiting?: (technically does not depend on CONFIG_PREEMPT,
#endif
}
-/*
- * Idle thread specific functions to determine the need_resched
- * polling state.
- */
-#ifdef TIF_POLLING_NRFLAG
-static inline int tsk_is_polling(struct task_struct *p)
-{
- return test_tsk_thread_flag(p, TIF_POLLING_NRFLAG);
-}
-
-static inline void __current_set_polling(void)
-{
- set_thread_flag(TIF_POLLING_NRFLAG);
-}
-
-static inline bool __must_check current_set_polling_and_test(void)
-{
- __current_set_polling();
-
- /*
- * Polling state must be visible before we test NEED_RESCHED,
- * paired by resched_curr()
- */
- smp_mb__after_atomic();
-
- return unlikely(tif_need_resched());
-}
-
-static inline void __current_clr_polling(void)
-{
- clear_thread_flag(TIF_POLLING_NRFLAG);
-}
-
-static inline bool __must_check current_clr_polling_and_test(void)
-{
- __current_clr_polling();
-
- /*
- * Polling state must be visible before we test NEED_RESCHED,
- * paired by resched_curr()
- */
- smp_mb__after_atomic();
-
- return unlikely(tif_need_resched());
-}
-
-#else
-static inline int tsk_is_polling(struct task_struct *p) { return 0; }
-static inline void __current_set_polling(void) { }
-static inline void __current_clr_polling(void) { }
-
-static inline bool __must_check current_set_polling_and_test(void)
-{
- return unlikely(tif_need_resched());
-}
-static inline bool __must_check current_clr_polling_and_test(void)
-{
- return unlikely(tif_need_resched());
-}
-#endif
-
-static inline void current_clr_polling(void)
-{
- __current_clr_polling();
-
- /*
- * Ensure we check TIF_NEED_RESCHED after we clear the polling bit.
- * Once the bit is cleared, we'll get IPIs with every new
- * TIF_NEED_RESCHED and the IPI handler, scheduler_ipi(), will also
- * fold.
- */
- smp_mb(); /* paired with resched_curr() */
-
- preempt_fold_need_resched();
-}
-
static __always_inline bool need_resched(void)
{
return unlikely(tif_need_resched());
}
-/*
- * Thread group CPU time accounting.
- */
-void thread_group_cputime(struct task_struct *tsk, struct task_cputime *times);
-void thread_group_cputimer(struct task_struct *tsk, struct task_cputime *times);
-
-/*
- * Reevaluate whether the task has signals pending delivery.
- * Wake the task if so.
- * This is required every time the blocked sigset_t changes.
- * callers must hold sighand->siglock.
- */
-extern void recalc_sigpending_and_wake(struct task_struct *t);
-extern void recalc_sigpending(void);
-
-extern void signal_wake_up_state(struct task_struct *t, unsigned int state);
-
-static inline void signal_wake_up(struct task_struct *t, bool resume)
-{
- signal_wake_up_state(t, resume ? TASK_WAKEKILL : 0);
-}
-static inline void ptrace_signal_wake_up(struct task_struct *t, bool resume)
-{
- signal_wake_up_state(t, resume ? __TASK_TRACED : 0);
-}
-
/*
* Wrappers for p->thread_info->cpu access. No-op on UP.
*/
#endif
}
-static inline int task_node(const struct task_struct *p)
-{
- return cpu_to_node(task_cpu(p));
-}
-
extern void set_task_cpu(struct task_struct *p, unsigned int cpu);
#else
extern long sched_setaffinity(pid_t pid, const struct cpumask *new_mask);
extern long sched_getaffinity(pid_t pid, struct cpumask *mask);
-#ifdef CONFIG_CGROUP_SCHED
-extern struct task_group root_task_group;
-#endif /* CONFIG_CGROUP_SCHED */
-
-extern int task_can_switch_user(struct user_struct *up,
- struct task_struct *tsk);
-
-#ifdef CONFIG_TASK_XACCT
-static inline void add_rchar(struct task_struct *tsk, ssize_t amt)
-{
- tsk->ioac.rchar += amt;
-}
-
-static inline void add_wchar(struct task_struct *tsk, ssize_t amt)
-{
- tsk->ioac.wchar += amt;
-}
-
-static inline void inc_syscr(struct task_struct *tsk)
-{
- tsk->ioac.syscr++;
-}
-
-static inline void inc_syscw(struct task_struct *tsk)
-{
- tsk->ioac.syscw++;
-}
-#else
-static inline void add_rchar(struct task_struct *tsk, ssize_t amt)
-{
-}
-
-static inline void add_wchar(struct task_struct *tsk, ssize_t amt)
-{
-}
-
-static inline void inc_syscr(struct task_struct *tsk)
-{
-}
-
-static inline void inc_syscw(struct task_struct *tsk)
-{
-}
-#endif
-
#ifndef TASK_SIZE_OF
#define TASK_SIZE_OF(tsk) TASK_SIZE
#endif
-#ifdef CONFIG_MEMCG
-extern void mm_update_next_owner(struct mm_struct *mm);
-#else
-static inline void mm_update_next_owner(struct mm_struct *mm)
-{
-}
-#endif /* CONFIG_MEMCG */
-
-static inline unsigned long task_rlimit(const struct task_struct *tsk,
- unsigned int limit)
-{
- return READ_ONCE(tsk->signal->rlim[limit].rlim_cur);
-}
-
-static inline unsigned long task_rlimit_max(const struct task_struct *tsk,
- unsigned int limit)
-{
- return READ_ONCE(tsk->signal->rlim[limit].rlim_max);
-}
-
-static inline unsigned long rlimit(unsigned int limit)
-{
- return task_rlimit(current, limit);
-}
-
-static inline unsigned long rlimit_max(unsigned int limit)
-{
- return task_rlimit_max(current, limit);
-}
-
-#define SCHED_CPUFREQ_RT (1U << 0)
-#define SCHED_CPUFREQ_DL (1U << 1)
-#define SCHED_CPUFREQ_IOWAIT (1U << 2)
-
-#define SCHED_CPUFREQ_RT_DL (SCHED_CPUFREQ_RT | SCHED_CPUFREQ_DL)
-
-#ifdef CONFIG_CPU_FREQ
-struct update_util_data {
- void (*func)(struct update_util_data *data, u64 time, unsigned int flags);
-};
-
-void cpufreq_add_update_util_hook(int cpu, struct update_util_data *data,
- void (*func)(struct update_util_data *data, u64 time,
- unsigned int flags));
-void cpufreq_remove_update_util_hook(int cpu);
-#endif /* CONFIG_CPU_FREQ */
-
#endif
--- /dev/null
+#ifndef _LINUX_SCHED_AUTOGROUP_H
+#define _LINUX_SCHED_AUTOGROUP_H
+
+struct signal_struct;
+struct task_struct;
+struct task_group;
+struct seq_file;
+
+#ifdef CONFIG_SCHED_AUTOGROUP
+extern void sched_autogroup_create_attach(struct task_struct *p);
+extern void sched_autogroup_detach(struct task_struct *p);
+extern void sched_autogroup_fork(struct signal_struct *sig);
+extern void sched_autogroup_exit(struct signal_struct *sig);
+extern void sched_autogroup_exit_task(struct task_struct *p);
+#ifdef CONFIG_PROC_FS
+extern void proc_sched_autogroup_show_task(struct task_struct *p, struct seq_file *m);
+extern int proc_sched_autogroup_set_nice(struct task_struct *p, int nice);
+#endif
+#else
+static inline void sched_autogroup_create_attach(struct task_struct *p) { }
+static inline void sched_autogroup_detach(struct task_struct *p) { }
+static inline void sched_autogroup_fork(struct signal_struct *sig) { }
+static inline void sched_autogroup_exit(struct signal_struct *sig) { }
+static inline void sched_autogroup_exit_task(struct task_struct *p) { }
+#endif
+
+#ifdef CONFIG_CGROUP_SCHED
+extern struct task_group root_task_group;
+#endif /* CONFIG_CGROUP_SCHED */
+
+#endif /* _LINUX_SCHED_AUTOGROUP_H */
--- /dev/null
+#ifndef _LINUX_SCHED_CLOCK_H
+#define _LINUX_SCHED_CLOCK_H
+
+#include <linux/smp.h>
+
+/*
+ * Do not use outside of architecture code which knows its limitations.
+ *
+ * sched_clock() has no promise of monotonicity or bounded drift between
+ * CPUs, use (which you should not) requires disabling IRQs.
+ *
+ * Please use one of the three interfaces below.
+ */
+extern unsigned long long notrace sched_clock(void);
+
+/*
+ * See the comment in kernel/sched/clock.c
+ */
+extern u64 running_clock(void);
+extern u64 sched_clock_cpu(int cpu);
+
+
+extern void sched_clock_init(void);
+
+#ifndef CONFIG_HAVE_UNSTABLE_SCHED_CLOCK
+static inline void sched_clock_init_late(void)
+{
+}
+
+static inline void sched_clock_tick(void)
+{
+}
+
+static inline void clear_sched_clock_stable(void)
+{
+}
+
+static inline void sched_clock_idle_sleep_event(void)
+{
+}
+
+static inline void sched_clock_idle_wakeup_event(u64 delta_ns)
+{
+}
+
+static inline u64 cpu_clock(int cpu)
+{
+ return sched_clock();
+}
+
+static inline u64 local_clock(void)
+{
+ return sched_clock();
+}
+#else
+extern void sched_clock_init_late(void);
+/*
+ * Architectures can set this to 1 if they have specified
+ * CONFIG_HAVE_UNSTABLE_SCHED_CLOCK in their arch Kconfig,
+ * but then during bootup it turns out that sched_clock()
+ * is reliable after all:
+ */
+extern int sched_clock_stable(void);
+extern void clear_sched_clock_stable(void);
+
+extern void sched_clock_tick(void);
+extern void sched_clock_idle_sleep_event(void);
+extern void sched_clock_idle_wakeup_event(u64 delta_ns);
+
+/*
+ * As outlined in clock.c, provides a fast, high resolution, nanosecond
+ * time source that is monotonic per cpu argument and has bounded drift
+ * between cpus.
+ *
+ * ######################### BIG FAT WARNING ##########################
+ * # when comparing cpu_clock(i) to cpu_clock(j) for i != j, time can #
+ * # go backwards !! #
+ * ####################################################################
+ */
+static inline u64 cpu_clock(int cpu)
+{
+ return sched_clock_cpu(cpu);
+}
+
+static inline u64 local_clock(void)
+{
+ return sched_clock_cpu(raw_smp_processor_id());
+}
+#endif
+
+#ifdef CONFIG_IRQ_TIME_ACCOUNTING
+/*
+ * An i/f to runtime opt-in for irq time accounting based off of sched_clock.
+ * The reason for this explicit opt-in is not to have perf penalty with
+ * slow sched_clocks.
+ */
+extern void enable_sched_clock_irqtime(void);
+extern void disable_sched_clock_irqtime(void);
+#else
+static inline void enable_sched_clock_irqtime(void) {}
+static inline void disable_sched_clock_irqtime(void) {}
+#endif
+
+#endif /* _LINUX_SCHED_CLOCK_H */
--- /dev/null
+#ifndef _LINUX_SCHED_COREDUMP_H
+#define _LINUX_SCHED_COREDUMP_H
+
+#include <linux/mm_types.h>
+
+#define SUID_DUMP_DISABLE 0 /* No setuid dumping */
+#define SUID_DUMP_USER 1 /* Dump as user of process */
+#define SUID_DUMP_ROOT 2 /* Dump as root */
+
+/* mm flags */
+
+/* for SUID_DUMP_* above */
+#define MMF_DUMPABLE_BITS 2
+#define MMF_DUMPABLE_MASK ((1 << MMF_DUMPABLE_BITS) - 1)
+
+extern void set_dumpable(struct mm_struct *mm, int value);
+/*
+ * This returns the actual value of the suid_dumpable flag. For things
+ * that are using this for checking for privilege transitions, it must
+ * test against SUID_DUMP_USER rather than treating it as a boolean
+ * value.
+ */
+static inline int __get_dumpable(unsigned long mm_flags)
+{
+ return mm_flags & MMF_DUMPABLE_MASK;
+}
+
+static inline int get_dumpable(struct mm_struct *mm)
+{
+ return __get_dumpable(mm->flags);
+}
+
+/* coredump filter bits */
+#define MMF_DUMP_ANON_PRIVATE 2
+#define MMF_DUMP_ANON_SHARED 3
+#define MMF_DUMP_MAPPED_PRIVATE 4
+#define MMF_DUMP_MAPPED_SHARED 5
+#define MMF_DUMP_ELF_HEADERS 6
+#define MMF_DUMP_HUGETLB_PRIVATE 7
+#define MMF_DUMP_HUGETLB_SHARED 8
+#define MMF_DUMP_DAX_PRIVATE 9
+#define MMF_DUMP_DAX_SHARED 10
+
+#define MMF_DUMP_FILTER_SHIFT MMF_DUMPABLE_BITS
+#define MMF_DUMP_FILTER_BITS 9
+#define MMF_DUMP_FILTER_MASK \
+ (((1 << MMF_DUMP_FILTER_BITS) - 1) << MMF_DUMP_FILTER_SHIFT)
+#define MMF_DUMP_FILTER_DEFAULT \
+ ((1 << MMF_DUMP_ANON_PRIVATE) | (1 << MMF_DUMP_ANON_SHARED) |\
+ (1 << MMF_DUMP_HUGETLB_PRIVATE) | MMF_DUMP_MASK_DEFAULT_ELF)
+
+#ifdef CONFIG_CORE_DUMP_DEFAULT_ELF_HEADERS
+# define MMF_DUMP_MASK_DEFAULT_ELF (1 << MMF_DUMP_ELF_HEADERS)
+#else
+# define MMF_DUMP_MASK_DEFAULT_ELF 0
+#endif
+ /* leave room for more dump flags */
+#define MMF_VM_MERGEABLE 16 /* KSM may merge identical pages */
+#define MMF_VM_HUGEPAGE 17 /* set when VM_HUGEPAGE is set on vma */
+/*
+ * This one-shot flag is dropped due to necessity of changing exe once again
+ * on NFS restore
+ */
+//#define MMF_EXE_FILE_CHANGED 18 /* see prctl_set_mm_exe_file() */
+
+#define MMF_HAS_UPROBES 19 /* has uprobes */
+#define MMF_RECALC_UPROBES 20 /* MMF_HAS_UPROBES can be wrong */
+#define MMF_OOM_SKIP 21 /* mm is of no interest for the OOM killer */
+#define MMF_UNSTABLE 22 /* mm is unstable for copy_from_user */
+#define MMF_HUGE_ZERO_PAGE 23 /* mm has ever used the global huge zero page */
+
+#define MMF_INIT_MASK (MMF_DUMPABLE_MASK | MMF_DUMP_FILTER_MASK)
+
+#endif /* _LINUX_SCHED_COREDUMP_H */
--- /dev/null
+#ifndef _LINUX_SCHED_CPUFREQ_H
+#define _LINUX_SCHED_CPUFREQ_H
+
+#include <linux/types.h>
+
+/*
+ * Interface between cpufreq drivers and the scheduler:
+ */
+
+#define SCHED_CPUFREQ_RT (1U << 0)
+#define SCHED_CPUFREQ_DL (1U << 1)
+#define SCHED_CPUFREQ_IOWAIT (1U << 2)
+
+#define SCHED_CPUFREQ_RT_DL (SCHED_CPUFREQ_RT | SCHED_CPUFREQ_DL)
+
+#ifdef CONFIG_CPU_FREQ
+struct update_util_data {
+ void (*func)(struct update_util_data *data, u64 time, unsigned int flags);
+};
+
+void cpufreq_add_update_util_hook(int cpu, struct update_util_data *data,
+ void (*func)(struct update_util_data *data, u64 time,
+ unsigned int flags));
+void cpufreq_remove_update_util_hook(int cpu);
+#endif /* CONFIG_CPU_FREQ */
+
+#endif /* _LINUX_SCHED_CPUFREQ_H */
--- /dev/null
+#ifndef _LINUX_SCHED_CPUTIME_H
+#define _LINUX_SCHED_CPUTIME_H
+
+#include <linux/sched/signal.h>
+
+/*
+ * cputime accounting APIs:
+ */
+
+#ifdef CONFIG_VIRT_CPU_ACCOUNTING_NATIVE
+#include <asm/cputime.h>
+
+#ifndef cputime_to_nsecs
+# define cputime_to_nsecs(__ct) \
+ (cputime_to_usecs(__ct) * NSEC_PER_USEC)
+#endif
+#endif /* CONFIG_VIRT_CPU_ACCOUNTING_NATIVE */
+
+#ifdef CONFIG_VIRT_CPU_ACCOUNTING_GEN
+extern void task_cputime(struct task_struct *t,
+ u64 *utime, u64 *stime);
+extern u64 task_gtime(struct task_struct *t);
+#else
+static inline void task_cputime(struct task_struct *t,
+ u64 *utime, u64 *stime)
+{
+ *utime = t->utime;
+ *stime = t->stime;
+}
+
+static inline u64 task_gtime(struct task_struct *t)
+{
+ return t->gtime;
+}
+#endif
+
+#ifdef CONFIG_ARCH_HAS_SCALED_CPUTIME
+static inline void task_cputime_scaled(struct task_struct *t,
+ u64 *utimescaled,
+ u64 *stimescaled)
+{
+ *utimescaled = t->utimescaled;
+ *stimescaled = t->stimescaled;
+}
+#else
+static inline void task_cputime_scaled(struct task_struct *t,
+ u64 *utimescaled,
+ u64 *stimescaled)
+{
+ task_cputime(t, utimescaled, stimescaled);
+}
+#endif
+
+extern void task_cputime_adjusted(struct task_struct *p, u64 *ut, u64 *st);
+extern void thread_group_cputime_adjusted(struct task_struct *p, u64 *ut, u64 *st);
+
+
+/*
+ * Thread group CPU time accounting.
+ */
+void thread_group_cputime(struct task_struct *tsk, struct task_cputime *times);
+void thread_group_cputimer(struct task_struct *tsk, struct task_cputime *times);
+
+
+/*
+ * The following are functions that support scheduler-internal time accounting.
+ * These functions are generally called at the timer tick. None of this depends
+ * on CONFIG_SCHEDSTATS.
+ */
+
+/**
+ * get_running_cputimer - return &tsk->signal->cputimer if cputimer is running
+ *
+ * @tsk: Pointer to target task.
+ */
+#ifdef CONFIG_POSIX_TIMERS
+static inline
+struct thread_group_cputimer *get_running_cputimer(struct task_struct *tsk)
+{
+ struct thread_group_cputimer *cputimer = &tsk->signal->cputimer;
+
+ /* Check if cputimer isn't running. This is accessed without locking. */
+ if (!READ_ONCE(cputimer->running))
+ return NULL;
+
+ /*
+ * After we flush the task's sum_exec_runtime to sig->sum_sched_runtime
+ * in __exit_signal(), we won't account to the signal struct further
+ * cputime consumed by that task, even though the task can still be
+ * ticking after __exit_signal().
+ *
+ * In order to keep a consistent behaviour between thread group cputime
+ * and thread group cputimer accounting, lets also ignore the cputime
+ * elapsing after __exit_signal() in any thread group timer running.
+ *
+ * This makes sure that POSIX CPU clocks and timers are synchronized, so
+ * that a POSIX CPU timer won't expire while the corresponding POSIX CPU
+ * clock delta is behind the expiring timer value.
+ */
+ if (unlikely(!tsk->sighand))
+ return NULL;
+
+ return cputimer;
+}
+#else
+static inline
+struct thread_group_cputimer *get_running_cputimer(struct task_struct *tsk)
+{
+ return NULL;
+}
+#endif
+
+/**
+ * account_group_user_time - Maintain utime for a thread group.
+ *
+ * @tsk: Pointer to task structure.
+ * @cputime: Time value by which to increment the utime field of the
+ * thread_group_cputime structure.
+ *
+ * If thread group time is being maintained, get the structure for the
+ * running CPU and update the utime field there.
+ */
+static inline void account_group_user_time(struct task_struct *tsk,
+ u64 cputime)
+{
+ struct thread_group_cputimer *cputimer = get_running_cputimer(tsk);
+
+ if (!cputimer)
+ return;
+
+ atomic64_add(cputime, &cputimer->cputime_atomic.utime);
+}
+
+/**
+ * account_group_system_time - Maintain stime for a thread group.
+ *
+ * @tsk: Pointer to task structure.
+ * @cputime: Time value by which to increment the stime field of the
+ * thread_group_cputime structure.
+ *
+ * If thread group time is being maintained, get the structure for the
+ * running CPU and update the stime field there.
+ */
+static inline void account_group_system_time(struct task_struct *tsk,
+ u64 cputime)
+{
+ struct thread_group_cputimer *cputimer = get_running_cputimer(tsk);
+
+ if (!cputimer)
+ return;
+
+ atomic64_add(cputime, &cputimer->cputime_atomic.stime);
+}
+
+/**
+ * account_group_exec_runtime - Maintain exec runtime for a thread group.
+ *
+ * @tsk: Pointer to task structure.
+ * @ns: Time value by which to increment the sum_exec_runtime field
+ * of the thread_group_cputime structure.
+ *
+ * If thread group time is being maintained, get the structure for the
+ * running CPU and update the sum_exec_runtime field there.
+ */
+static inline void account_group_exec_runtime(struct task_struct *tsk,
+ unsigned long long ns)
+{
+ struct thread_group_cputimer *cputimer = get_running_cputimer(tsk);
+
+ if (!cputimer)
+ return;
+
+ atomic64_add(ns, &cputimer->cputime_atomic.sum_exec_runtime);
+}
+
+static inline void prev_cputime_init(struct prev_cputime *prev)
+{
+#ifndef CONFIG_VIRT_CPU_ACCOUNTING_NATIVE
+ prev->utime = prev->stime = 0;
+ raw_spin_lock_init(&prev->lock);
+#endif
+}
+
+extern unsigned long long
+task_sched_runtime(struct task_struct *task);
+
+#endif /* _LINUX_SCHED_CPUTIME_H */
-#ifndef _SCHED_DEADLINE_H
-#define _SCHED_DEADLINE_H
+#ifndef _LINUX_SCHED_DEADLINE_H
+#define _LINUX_SCHED_DEADLINE_H
+
+#include <linux/sched.h>
/*
* SCHED_DEADLINE tasks has negative priorities, reflecting
return (s64)(a - b) < 0;
}
-#endif /* _SCHED_DEADLINE_H */
+#endif /* _LINUX_SCHED_DEADLINE_H */
--- /dev/null
+#ifndef _LINUX_SCHED_DEBUG_H
+#define _LINUX_SCHED_DEBUG_H
+
+/*
+ * Various scheduler/task debugging interfaces:
+ */
+
+struct task_struct;
+
+extern void dump_cpu_task(int cpu);
+
+/*
+ * Only dump TASK_* tasks. (0 for all tasks)
+ */
+extern void show_state_filter(unsigned long state_filter);
+
+static inline void show_state(void)
+{
+ show_state_filter(0);
+}
+
+struct pt_regs;
+
+extern void show_regs(struct pt_regs *);
+
+/*
+ * TASK is a pointer to the task whose backtrace we want to see (or NULL for current
+ * task), SP is the stack pointer of the first frame that should be shown in the back
+ * trace (or NULL if the entire call-chain of the task should be shown).
+ */
+extern void show_stack(struct task_struct *task, unsigned long *sp);
+
+extern void sched_show_task(struct task_struct *p);
+
+#ifdef CONFIG_SCHED_DEBUG
+struct seq_file;
+extern void proc_sched_show_task(struct task_struct *p, struct seq_file *m);
+extern void proc_sched_set_task(struct task_struct *p);
+#endif
+
+/* Attach to any functions which should be ignored in wchan output. */
+#define __sched __attribute__((__section__(".sched.text")))
+
+/* Linker adds these: start and end of __sched functions */
+extern char __sched_text_start[], __sched_text_end[];
+
+/* Is this address in the __sched functions? */
+extern int in_sched_functions(unsigned long addr);
+
+#endif /* _LINUX_SCHED_DEBUG_H */
--- /dev/null
+#ifndef _LINUX_SCHED_HOTPLUG_H
+#define _LINUX_SCHED_HOTPLUG_H
+
+/*
+ * Scheduler interfaces for hotplug CPU support:
+ */
+
+extern int sched_cpu_starting(unsigned int cpu);
+extern int sched_cpu_activate(unsigned int cpu);
+extern int sched_cpu_deactivate(unsigned int cpu);
+
+#ifdef CONFIG_HOTPLUG_CPU
+extern int sched_cpu_dying(unsigned int cpu);
+#else
+# define sched_cpu_dying NULL
+#endif
+
+#ifdef CONFIG_HOTPLUG_CPU
+extern void idle_task_exit(void);
+#else
+static inline void idle_task_exit(void) {}
+#endif
+
+#endif /* _LINUX_SCHED_HOTPLUG_H */
--- /dev/null
+#ifndef _LINUX_SCHED_IDLE_H
+#define _LINUX_SCHED_IDLE_H
+
+#include <linux/sched.h>
+
+enum cpu_idle_type {
+ CPU_IDLE,
+ CPU_NOT_IDLE,
+ CPU_NEWLY_IDLE,
+ CPU_MAX_IDLE_TYPES
+};
+
+extern void wake_up_if_idle(int cpu);
+
+/*
+ * Idle thread specific functions to determine the need_resched
+ * polling state.
+ */
+#ifdef TIF_POLLING_NRFLAG
+
+static inline void __current_set_polling(void)
+{
+ set_thread_flag(TIF_POLLING_NRFLAG);
+}
+
+static inline bool __must_check current_set_polling_and_test(void)
+{
+ __current_set_polling();
+
+ /*
+ * Polling state must be visible before we test NEED_RESCHED,
+ * paired by resched_curr()
+ */
+ smp_mb__after_atomic();
+
+ return unlikely(tif_need_resched());
+}
+
+static inline void __current_clr_polling(void)
+{
+ clear_thread_flag(TIF_POLLING_NRFLAG);
+}
+
+static inline bool __must_check current_clr_polling_and_test(void)
+{
+ __current_clr_polling();
+
+ /*
+ * Polling state must be visible before we test NEED_RESCHED,
+ * paired by resched_curr()
+ */
+ smp_mb__after_atomic();
+
+ return unlikely(tif_need_resched());
+}
+
+#else
+static inline void __current_set_polling(void) { }
+static inline void __current_clr_polling(void) { }
+
+static inline bool __must_check current_set_polling_and_test(void)
+{
+ return unlikely(tif_need_resched());
+}
+static inline bool __must_check current_clr_polling_and_test(void)
+{
+ return unlikely(tif_need_resched());
+}
+#endif
+
+static inline void current_clr_polling(void)
+{
+ __current_clr_polling();
+
+ /*
+ * Ensure we check TIF_NEED_RESCHED after we clear the polling bit.
+ * Once the bit is cleared, we'll get IPIs with every new
+ * TIF_NEED_RESCHED and the IPI handler, scheduler_ipi(), will also
+ * fold.
+ */
+ smp_mb(); /* paired with resched_curr() */
+
+ preempt_fold_need_resched();
+}
+
+#endif /* _LINUX_SCHED_IDLE_H */
--- /dev/null
+#ifndef _LINUX_SCHED_INIT_H
+#define _LINUX_SCHED_INIT_H
+
+/*
+ * Scheduler init related prototypes:
+ */
+
+extern void sched_init(void);
+extern void sched_init_smp(void);
+
+#endif /* _LINUX_SCHED_INIT_H */
--- /dev/null
+#ifndef _LINUX_SCHED_JOBCTL_H
+#define _LINUX_SCHED_JOBCTL_H
+
+#include <linux/types.h>
+
+struct task_struct;
+
+/*
+ * task->jobctl flags
+ */
+#define JOBCTL_STOP_SIGMASK 0xffff /* signr of the last group stop */
+
+#define JOBCTL_STOP_DEQUEUED_BIT 16 /* stop signal dequeued */
+#define JOBCTL_STOP_PENDING_BIT 17 /* task should stop for group stop */
+#define JOBCTL_STOP_CONSUME_BIT 18 /* consume group stop count */
+#define JOBCTL_TRAP_STOP_BIT 19 /* trap for STOP */
+#define JOBCTL_TRAP_NOTIFY_BIT 20 /* trap for NOTIFY */
+#define JOBCTL_TRAPPING_BIT 21 /* switching to TRACED */
+#define JOBCTL_LISTENING_BIT 22 /* ptracer is listening for events */
+
+#define JOBCTL_STOP_DEQUEUED (1UL << JOBCTL_STOP_DEQUEUED_BIT)
+#define JOBCTL_STOP_PENDING (1UL << JOBCTL_STOP_PENDING_BIT)
+#define JOBCTL_STOP_CONSUME (1UL << JOBCTL_STOP_CONSUME_BIT)
+#define JOBCTL_TRAP_STOP (1UL << JOBCTL_TRAP_STOP_BIT)
+#define JOBCTL_TRAP_NOTIFY (1UL << JOBCTL_TRAP_NOTIFY_BIT)
+#define JOBCTL_TRAPPING (1UL << JOBCTL_TRAPPING_BIT)
+#define JOBCTL_LISTENING (1UL << JOBCTL_LISTENING_BIT)
+
+#define JOBCTL_TRAP_MASK (JOBCTL_TRAP_STOP | JOBCTL_TRAP_NOTIFY)
+#define JOBCTL_PENDING_MASK (JOBCTL_STOP_PENDING | JOBCTL_TRAP_MASK)
+
+extern bool task_set_jobctl_pending(struct task_struct *task, unsigned long mask);
+extern void task_clear_jobctl_trapping(struct task_struct *task);
+extern void task_clear_jobctl_pending(struct task_struct *task, unsigned long mask);
+
+#endif /* _LINUX_SCHED_JOBCTL_H */
--- /dev/null
+#ifndef _LINUX_SCHED_LOADAVG_H
+#define _LINUX_SCHED_LOADAVG_H
+
+/*
+ * These are the constant used to fake the fixed-point load-average
+ * counting. Some notes:
+ * - 11 bit fractions expand to 22 bits by the multiplies: this gives
+ * a load-average precision of 10 bits integer + 11 bits fractional
+ * - if you want to count load-averages more often, you need more
+ * precision, or rounding will get you. With 2-second counting freq,
+ * the EXP_n values would be 1981, 2034 and 2043 if still using only
+ * 11 bit fractions.
+ */
+extern unsigned long avenrun[]; /* Load averages */
+extern void get_avenrun(unsigned long *loads, unsigned long offset, int shift);
+
+#define FSHIFT 11 /* nr of bits of precision */
+#define FIXED_1 (1<<FSHIFT) /* 1.0 as fixed-point */
+#define LOAD_FREQ (5*HZ+1) /* 5 sec intervals */
+#define EXP_1 1884 /* 1/exp(5sec/1min) as fixed-point */
+#define EXP_5 2014 /* 1/exp(5sec/5min) */
+#define EXP_15 2037 /* 1/exp(5sec/15min) */
+
+#define CALC_LOAD(load,exp,n) \
+ load *= exp; \
+ load += n*(FIXED_1-exp); \
+ load >>= FSHIFT;
+
+extern void calc_global_load(unsigned long ticks);
+
+#endif /* _LINUX_SCHED_LOADAVG_H */
--- /dev/null
+#ifndef _LINUX_SCHED_MM_H
+#define _LINUX_SCHED_MM_H
+
+#include <linux/kernel.h>
+#include <linux/atomic.h>
+#include <linux/sched.h>
+#include <linux/mm_types.h>
+#include <linux/gfp.h>
+
+/*
+ * Routines for handling mm_structs
+ */
+extern struct mm_struct * mm_alloc(void);
+
+/**
+ * mmgrab() - Pin a &struct mm_struct.
+ * @mm: The &struct mm_struct to pin.
+ *
+ * Make sure that @mm will not get freed even after the owning task
+ * exits. This doesn't guarantee that the associated address space
+ * will still exist later on and mmget_not_zero() has to be used before
+ * accessing it.
+ *
+ * This is a preferred way to to pin @mm for a longer/unbounded amount
+ * of time.
+ *
+ * Use mmdrop() to release the reference acquired by mmgrab().
+ *
+ * See also <Documentation/vm/active_mm.txt> for an in-depth explanation
+ * of &mm_struct.mm_count vs &mm_struct.mm_users.
+ */
+static inline void mmgrab(struct mm_struct *mm)
+{
+ atomic_inc(&mm->mm_count);
+}
+
+/* mmdrop drops the mm and the page tables */
+extern void __mmdrop(struct mm_struct *);
+static inline void mmdrop(struct mm_struct *mm)
+{
+ if (unlikely(atomic_dec_and_test(&mm->mm_count)))
+ __mmdrop(mm);
+}
+
+static inline void mmdrop_async_fn(struct work_struct *work)
+{
+ struct mm_struct *mm = container_of(work, struct mm_struct, async_put_work);
+ __mmdrop(mm);
+}
+
+static inline void mmdrop_async(struct mm_struct *mm)
+{
+ if (unlikely(atomic_dec_and_test(&mm->mm_count))) {
+ INIT_WORK(&mm->async_put_work, mmdrop_async_fn);
+ schedule_work(&mm->async_put_work);
+ }
+}
+
+/**
+ * mmget() - Pin the address space associated with a &struct mm_struct.
+ * @mm: The address space to pin.
+ *
+ * Make sure that the address space of the given &struct mm_struct doesn't
+ * go away. This does not protect against parts of the address space being
+ * modified or freed, however.
+ *
+ * Never use this function to pin this address space for an
+ * unbounded/indefinite amount of time.
+ *
+ * Use mmput() to release the reference acquired by mmget().
+ *
+ * See also <Documentation/vm/active_mm.txt> for an in-depth explanation
+ * of &mm_struct.mm_count vs &mm_struct.mm_users.
+ */
+static inline void mmget(struct mm_struct *mm)
+{
+ atomic_inc(&mm->mm_users);
+}
+
+static inline bool mmget_not_zero(struct mm_struct *mm)
+{
+ return atomic_inc_not_zero(&mm->mm_users);
+}
+
+/* mmput gets rid of the mappings and all user-space */
+extern void mmput(struct mm_struct *);
+#ifdef CONFIG_MMU
+/* same as above but performs the slow path from the async context. Can
+ * be called from the atomic context as well
+ */
+extern void mmput_async(struct mm_struct *);
+#endif
+
+/* Grab a reference to a task's mm, if it is not already going away */
+extern struct mm_struct *get_task_mm(struct task_struct *task);
+/*
+ * Grab a reference to a task's mm, if it is not already going away
+ * and ptrace_may_access with the mode parameter passed to it
+ * succeeds.
+ */
+extern struct mm_struct *mm_access(struct task_struct *task, unsigned int mode);
+/* Remove the current tasks stale references to the old mm_struct */
+extern void mm_release(struct task_struct *, struct mm_struct *);
+
+#ifdef CONFIG_MEMCG
+extern void mm_update_next_owner(struct mm_struct *mm);
+#else
+static inline void mm_update_next_owner(struct mm_struct *mm)
+{
+}
+#endif /* CONFIG_MEMCG */
+
+#ifdef CONFIG_MMU
+extern void arch_pick_mmap_layout(struct mm_struct *mm);
+extern unsigned long
+arch_get_unmapped_area(struct file *, unsigned long, unsigned long,
+ unsigned long, unsigned long);
+extern unsigned long
+arch_get_unmapped_area_topdown(struct file *filp, unsigned long addr,
+ unsigned long len, unsigned long pgoff,
+ unsigned long flags);
+#else
+static inline void arch_pick_mmap_layout(struct mm_struct *mm) {}
+#endif
+
+static inline bool in_vfork(struct task_struct *tsk)
+{
+ bool ret;
+
+ /*
+ * need RCU to access ->real_parent if CLONE_VM was used along with
+ * CLONE_PARENT.
+ *
+ * We check real_parent->mm == tsk->mm because CLONE_VFORK does not
+ * imply CLONE_VM
+ *
+ * CLONE_VFORK can be used with CLONE_PARENT/CLONE_THREAD and thus
+ * ->real_parent is not necessarily the task doing vfork(), so in
+ * theory we can't rely on task_lock() if we want to dereference it.
+ *
+ * And in this case we can't trust the real_parent->mm == tsk->mm
+ * check, it can be false negative. But we do not care, if init or
+ * another oom-unkillable task does this it should blame itself.
+ */
+ rcu_read_lock();
+ ret = tsk->vfork_done && tsk->real_parent->mm == tsk->mm;
+ rcu_read_unlock();
+
+ return ret;
+}
+
+/* __GFP_IO isn't allowed if PF_MEMALLOC_NOIO is set in current->flags
+ * __GFP_FS is also cleared as it implies __GFP_IO.
+ */
+static inline gfp_t memalloc_noio_flags(gfp_t flags)
+{
+ if (unlikely(current->flags & PF_MEMALLOC_NOIO))
+ flags &= ~(__GFP_IO | __GFP_FS);
+ return flags;
+}
+
+static inline unsigned int memalloc_noio_save(void)
+{
+ unsigned int flags = current->flags & PF_MEMALLOC_NOIO;
+ current->flags |= PF_MEMALLOC_NOIO;
+ return flags;
+}
+
+static inline void memalloc_noio_restore(unsigned int flags)
+{
+ current->flags = (current->flags & ~PF_MEMALLOC_NOIO) | flags;
+}
+
+#endif /* _LINUX_SCHED_MM_H */
--- /dev/null
+#ifndef _LINUX_SCHED_NOHZ_H
+#define _LINUX_SCHED_NOHZ_H
+
+/*
+ * This is the interface between the scheduler and nohz/dyntics:
+ */
+
+#if defined(CONFIG_SMP) && defined(CONFIG_NO_HZ_COMMON)
+extern void cpu_load_update_nohz_start(void);
+extern void cpu_load_update_nohz_stop(void);
+#else
+static inline void cpu_load_update_nohz_start(void) { }
+static inline void cpu_load_update_nohz_stop(void) { }
+#endif
+
+#if defined(CONFIG_SMP) && defined(CONFIG_NO_HZ_COMMON)
+extern void nohz_balance_enter_idle(int cpu);
+extern void set_cpu_sd_state_idle(void);
+extern int get_nohz_timer_target(void);
+#else
+static inline void nohz_balance_enter_idle(int cpu) { }
+static inline void set_cpu_sd_state_idle(void) { }
+#endif
+
+#ifdef CONFIG_NO_HZ_COMMON
+void calc_load_enter_idle(void);
+void calc_load_exit_idle(void);
+#else
+static inline void calc_load_enter_idle(void) { }
+static inline void calc_load_exit_idle(void) { }
+#endif /* CONFIG_NO_HZ_COMMON */
+
+#if defined(CONFIG_NO_HZ_COMMON) && defined(CONFIG_SMP)
+extern void wake_up_nohz_cpu(int cpu);
+#else
+static inline void wake_up_nohz_cpu(int cpu) { }
+#endif
+
+#ifdef CONFIG_NO_HZ_FULL
+extern u64 scheduler_tick_max_deferment(void);
+#endif
+
+#endif /* _LINUX_SCHED_NOHZ_H */
--- /dev/null
+#ifndef _LINUX_SCHED_NUMA_BALANCING_H
+#define _LINUX_SCHED_NUMA_BALANCING_H
+
+/*
+ * This is the interface between the scheduler and the MM that
+ * implements memory access pattern based NUMA-balancing:
+ */
+
+#include <linux/sched.h>
+
+#define TNF_MIGRATED 0x01
+#define TNF_NO_GROUP 0x02
+#define TNF_SHARED 0x04
+#define TNF_FAULT_LOCAL 0x08
+#define TNF_MIGRATE_FAIL 0x10
+
+#ifdef CONFIG_NUMA_BALANCING
+extern void task_numa_fault(int last_node, int node, int pages, int flags);
+extern pid_t task_numa_group_id(struct task_struct *p);
+extern void set_numabalancing_state(bool enabled);
+extern void task_numa_free(struct task_struct *p);
+extern bool should_numa_migrate_memory(struct task_struct *p, struct page *page,
+ int src_nid, int dst_cpu);
+#else
+static inline void task_numa_fault(int last_node, int node, int pages,
+ int flags)
+{
+}
+static inline pid_t task_numa_group_id(struct task_struct *p)
+{
+ return 0;
+}
+static inline void set_numabalancing_state(bool enabled)
+{
+}
+static inline void task_numa_free(struct task_struct *p)
+{
+}
+static inline bool should_numa_migrate_memory(struct task_struct *p,
+ struct page *page, int src_nid, int dst_cpu)
+{
+ return true;
+}
+#endif
+
+#endif /* _LINUX_SCHED_NUMA_BALANCING_H */
-#ifndef _SCHED_PRIO_H
-#define _SCHED_PRIO_H
+#ifndef _LINUX_SCHED_PRIO_H
+#define _LINUX_SCHED_PRIO_H
#define MAX_NICE 19
#define MIN_NICE -20
return (MAX_NICE - prio + 1);
}
-#endif /* _SCHED_PRIO_H */
+#endif /* _LINUX_SCHED_PRIO_H */
-#ifndef _SCHED_RT_H
-#define _SCHED_RT_H
+#ifndef _LINUX_SCHED_RT_H
+#define _LINUX_SCHED_RT_H
-#include <linux/sched/prio.h>
+#include <linux/sched.h>
+
+struct task_struct;
static inline int rt_prio(int prio)
{
*/
#define RR_TIMESLICE (100 * HZ / 1000)
-#endif /* _SCHED_RT_H */
+#endif /* _LINUX_SCHED_RT_H */
--- /dev/null
+#ifndef _LINUX_SCHED_SIGNAL_H
+#define _LINUX_SCHED_SIGNAL_H
+
+#include <linux/rculist.h>
+#include <linux/signal.h>
+#include <linux/sched.h>
+#include <linux/sched/jobctl.h>
+#include <linux/sched/task.h>
+#include <linux/cred.h>
+
+/*
+ * Types defining task->signal and task->sighand and APIs using them:
+ */
+
+struct sighand_struct {
+ atomic_t count;
+ struct k_sigaction action[_NSIG];
+ spinlock_t siglock;
+ wait_queue_head_t signalfd_wqh;
+};
+
+/*
+ * Per-process accounting stats:
+ */
+struct pacct_struct {
+ int ac_flag;
+ long ac_exitcode;
+ unsigned long ac_mem;
+ u64 ac_utime, ac_stime;
+ unsigned long ac_minflt, ac_majflt;
+};
+
+struct cpu_itimer {
+ u64 expires;
+ u64 incr;
+};
+
+/*
+ * This is the atomic variant of task_cputime, which can be used for
+ * storing and updating task_cputime statistics without locking.
+ */
+struct task_cputime_atomic {
+ atomic64_t utime;
+ atomic64_t stime;
+ atomic64_t sum_exec_runtime;
+};
+
+#define INIT_CPUTIME_ATOMIC \
+ (struct task_cputime_atomic) { \
+ .utime = ATOMIC64_INIT(0), \
+ .stime = ATOMIC64_INIT(0), \
+ .sum_exec_runtime = ATOMIC64_INIT(0), \
+ }
+/**
+ * struct thread_group_cputimer - thread group interval timer counts
+ * @cputime_atomic: atomic thread group interval timers.
+ * @running: true when there are timers running and
+ * @cputime_atomic receives updates.
+ * @checking_timer: true when a thread in the group is in the
+ * process of checking for thread group timers.
+ *
+ * This structure contains the version of task_cputime, above, that is
+ * used for thread group CPU timer calculations.
+ */
+struct thread_group_cputimer {
+ struct task_cputime_atomic cputime_atomic;
+ bool running;
+ bool checking_timer;
+};
+
+/*
+ * NOTE! "signal_struct" does not have its own
+ * locking, because a shared signal_struct always
+ * implies a shared sighand_struct, so locking
+ * sighand_struct is always a proper superset of
+ * the locking of signal_struct.
+ */
+struct signal_struct {
+ atomic_t sigcnt;
+ atomic_t live;
+ int nr_threads;
+ struct list_head thread_head;
+
+ wait_queue_head_t wait_chldexit; /* for wait4() */
+
+ /* current thread group signal load-balancing target: */
+ struct task_struct *curr_target;
+
+ /* shared signal handling: */
+ struct sigpending shared_pending;
+
+ /* thread group exit support */
+ int group_exit_code;
+ /* overloaded:
+ * - notify group_exit_task when ->count is equal to notify_count
+ * - everyone except group_exit_task is stopped during signal delivery
+ * of fatal signals, group_exit_task processes the signal.
+ */
+ int notify_count;
+ struct task_struct *group_exit_task;
+
+ /* thread group stop support, overloads group_exit_code too */
+ int group_stop_count;
+ unsigned int flags; /* see SIGNAL_* flags below */
+
+ /*
+ * PR_SET_CHILD_SUBREAPER marks a process, like a service
+ * manager, to re-parent orphan (double-forking) child processes
+ * to this process instead of 'init'. The service manager is
+ * able to receive SIGCHLD signals and is able to investigate
+ * the process until it calls wait(). All children of this
+ * process will inherit a flag if they should look for a
+ * child_subreaper process at exit.
+ */
+ unsigned int is_child_subreaper:1;
+ unsigned int has_child_subreaper:1;
+
+#ifdef CONFIG_POSIX_TIMERS
+
+ /* POSIX.1b Interval Timers */
+ int posix_timer_id;
+ struct list_head posix_timers;
+
+ /* ITIMER_REAL timer for the process */
+ struct hrtimer real_timer;
+ ktime_t it_real_incr;
+
+ /*
+ * ITIMER_PROF and ITIMER_VIRTUAL timers for the process, we use
+ * CPUCLOCK_PROF and CPUCLOCK_VIRT for indexing array as these
+ * values are defined to 0 and 1 respectively
+ */
+ struct cpu_itimer it[2];
+
+ /*
+ * Thread group totals for process CPU timers.
+ * See thread_group_cputimer(), et al, for details.
+ */
+ struct thread_group_cputimer cputimer;
+
+ /* Earliest-expiration cache. */
+ struct task_cputime cputime_expires;
+
+ struct list_head cpu_timers[3];
+
+#endif
+
+ struct pid *leader_pid;
+
+#ifdef CONFIG_NO_HZ_FULL
+ atomic_t tick_dep_mask;
+#endif
+
+ struct pid *tty_old_pgrp;
+
+ /* boolean value for session group leader */
+ int leader;
+
+ struct tty_struct *tty; /* NULL if no tty */
+
+#ifdef CONFIG_SCHED_AUTOGROUP
+ struct autogroup *autogroup;
+#endif
+ /*
+ * Cumulative resource counters for dead threads in the group,
+ * and for reaped dead child processes forked by this group.
+ * Live threads maintain their own counters and add to these
+ * in __exit_signal, except for the group leader.
+ */
+ seqlock_t stats_lock;
+ u64 utime, stime, cutime, cstime;
+ u64 gtime;
+ u64 cgtime;
+ struct prev_cputime prev_cputime;
+ unsigned long nvcsw, nivcsw, cnvcsw, cnivcsw;
+ unsigned long min_flt, maj_flt, cmin_flt, cmaj_flt;
+ unsigned long inblock, oublock, cinblock, coublock;
+ unsigned long maxrss, cmaxrss;
+ struct task_io_accounting ioac;
+
+ /*
+ * Cumulative ns of schedule CPU time fo dead threads in the
+ * group, not including a zombie group leader, (This only differs
+ * from jiffies_to_ns(utime + stime) if sched_clock uses something
+ * other than jiffies.)
+ */
+ unsigned long long sum_sched_runtime;
+
+ /*
+ * We don't bother to synchronize most readers of this at all,
+ * because there is no reader checking a limit that actually needs
+ * to get both rlim_cur and rlim_max atomically, and either one
+ * alone is a single word that can safely be read normally.
+ * getrlimit/setrlimit use task_lock(current->group_leader) to
+ * protect this instead of the siglock, because they really
+ * have no need to disable irqs.
+ */
+ struct rlimit rlim[RLIM_NLIMITS];
+
+#ifdef CONFIG_BSD_PROCESS_ACCT
+ struct pacct_struct pacct; /* per-process accounting information */
+#endif
+#ifdef CONFIG_TASKSTATS
+ struct taskstats *stats;
+#endif
+#ifdef CONFIG_AUDIT
+ unsigned audit_tty;
+ struct tty_audit_buf *tty_audit_buf;
+#endif
+
+ /*
+ * Thread is the potential origin of an oom condition; kill first on
+ * oom
+ */
+ bool oom_flag_origin;
+ short oom_score_adj; /* OOM kill score adjustment */
+ short oom_score_adj_min; /* OOM kill score adjustment min value.
+ * Only settable by CAP_SYS_RESOURCE. */
+ struct mm_struct *oom_mm; /* recorded mm when the thread group got
+ * killed by the oom killer */
+
+ struct mutex cred_guard_mutex; /* guard against foreign influences on
+ * credential calculations
+ * (notably. ptrace) */
+};
+
+/*
+ * Bits in flags field of signal_struct.
+ */
+#define SIGNAL_STOP_STOPPED 0x00000001 /* job control stop in effect */
+#define SIGNAL_STOP_CONTINUED 0x00000002 /* SIGCONT since WCONTINUED reap */
+#define SIGNAL_GROUP_EXIT 0x00000004 /* group exit in progress */
+#define SIGNAL_GROUP_COREDUMP 0x00000008 /* coredump in progress */
+/*
+ * Pending notifications to parent.
+ */
+#define SIGNAL_CLD_STOPPED 0x00000010
+#define SIGNAL_CLD_CONTINUED 0x00000020
+#define SIGNAL_CLD_MASK (SIGNAL_CLD_STOPPED|SIGNAL_CLD_CONTINUED)
+
+#define SIGNAL_UNKILLABLE 0x00000040 /* for init: ignore fatal signals */
+
+#define SIGNAL_STOP_MASK (SIGNAL_CLD_MASK | SIGNAL_STOP_STOPPED | \
+ SIGNAL_STOP_CONTINUED)
+
+static inline void signal_set_stop_flags(struct signal_struct *sig,
+ unsigned int flags)
+{
+ WARN_ON(sig->flags & (SIGNAL_GROUP_EXIT|SIGNAL_GROUP_COREDUMP));
+ sig->flags = (sig->flags & ~SIGNAL_STOP_MASK) | flags;
+}
+
+/* If true, all threads except ->group_exit_task have pending SIGKILL */
+static inline int signal_group_exit(const struct signal_struct *sig)
+{
+ return (sig->flags & SIGNAL_GROUP_EXIT) ||
+ (sig->group_exit_task != NULL);
+}
+
+extern void flush_signals(struct task_struct *);
+extern void ignore_signals(struct task_struct *);
+extern void flush_signal_handlers(struct task_struct *, int force_default);
+extern int dequeue_signal(struct task_struct *tsk, sigset_t *mask, siginfo_t *info);
+
+static inline int kernel_dequeue_signal(siginfo_t *info)
+{
+ struct task_struct *tsk = current;
+ siginfo_t __info;
+ int ret;
+
+ spin_lock_irq(&tsk->sighand->siglock);
+ ret = dequeue_signal(tsk, &tsk->blocked, info ?: &__info);
+ spin_unlock_irq(&tsk->sighand->siglock);
+
+ return ret;
+}
+
+static inline void kernel_signal_stop(void)
+{
+ spin_lock_irq(¤t->sighand->siglock);
+ if (current->jobctl & JOBCTL_STOP_DEQUEUED)
+ __set_current_state(TASK_STOPPED);
+ spin_unlock_irq(¤t->sighand->siglock);
+
+ schedule();
+}
+extern int send_sig_info(int, struct siginfo *, struct task_struct *);
+extern int force_sigsegv(int, struct task_struct *);
+extern int force_sig_info(int, struct siginfo *, struct task_struct *);
+extern int __kill_pgrp_info(int sig, struct siginfo *info, struct pid *pgrp);
+extern int kill_pid_info(int sig, struct siginfo *info, struct pid *pid);
+extern int kill_pid_info_as_cred(int, struct siginfo *, struct pid *,
+ const struct cred *, u32);
+extern int kill_pgrp(struct pid *pid, int sig, int priv);
+extern int kill_pid(struct pid *pid, int sig, int priv);
+extern int kill_proc_info(int, struct siginfo *, pid_t);
+extern __must_check bool do_notify_parent(struct task_struct *, int);
+extern void __wake_up_parent(struct task_struct *p, struct task_struct *parent);
+extern void force_sig(int, struct task_struct *);
+extern int send_sig(int, struct task_struct *, int);
+extern int zap_other_threads(struct task_struct *p);
+extern struct sigqueue *sigqueue_alloc(void);
+extern void sigqueue_free(struct sigqueue *);
+extern int send_sigqueue(struct sigqueue *, struct task_struct *, int group);
+extern int do_sigaction(int, struct k_sigaction *, struct k_sigaction *);
+
+static inline int restart_syscall(void)
+{
+ set_tsk_thread_flag(current, TIF_SIGPENDING);
+ return -ERESTARTNOINTR;
+}
+
+static inline int signal_pending(struct task_struct *p)
+{
+ return unlikely(test_tsk_thread_flag(p,TIF_SIGPENDING));
+}
+
+static inline int __fatal_signal_pending(struct task_struct *p)
+{
+ return unlikely(sigismember(&p->pending.signal, SIGKILL));
+}
+
+static inline int fatal_signal_pending(struct task_struct *p)
+{
+ return signal_pending(p) && __fatal_signal_pending(p);
+}
+
+static inline int signal_pending_state(long state, struct task_struct *p)
+{
+ if (!(state & (TASK_INTERRUPTIBLE | TASK_WAKEKILL)))
+ return 0;
+ if (!signal_pending(p))
+ return 0;
+
+ return (state & TASK_INTERRUPTIBLE) || __fatal_signal_pending(p);
+}
+
+/*
+ * Reevaluate whether the task has signals pending delivery.
+ * Wake the task if so.
+ * This is required every time the blocked sigset_t changes.
+ * callers must hold sighand->siglock.
+ */
+extern void recalc_sigpending_and_wake(struct task_struct *t);
+extern void recalc_sigpending(void);
+
+extern void signal_wake_up_state(struct task_struct *t, unsigned int state);
+
+static inline void signal_wake_up(struct task_struct *t, bool resume)
+{
+ signal_wake_up_state(t, resume ? TASK_WAKEKILL : 0);
+}
+static inline void ptrace_signal_wake_up(struct task_struct *t, bool resume)
+{
+ signal_wake_up_state(t, resume ? __TASK_TRACED : 0);
+}
+
+#ifdef TIF_RESTORE_SIGMASK
+/*
+ * Legacy restore_sigmask accessors. These are inefficient on
+ * SMP architectures because they require atomic operations.
+ */
+
+/**
+ * set_restore_sigmask() - make sure saved_sigmask processing gets done
+ *
+ * This sets TIF_RESTORE_SIGMASK and ensures that the arch signal code
+ * will run before returning to user mode, to process the flag. For
+ * all callers, TIF_SIGPENDING is already set or it's no harm to set
+ * it. TIF_RESTORE_SIGMASK need not be in the set of bits that the
+ * arch code will notice on return to user mode, in case those bits
+ * are scarce. We set TIF_SIGPENDING here to ensure that the arch
+ * signal code always gets run when TIF_RESTORE_SIGMASK is set.
+ */
+static inline void set_restore_sigmask(void)
+{
+ set_thread_flag(TIF_RESTORE_SIGMASK);
+ WARN_ON(!test_thread_flag(TIF_SIGPENDING));
+}
+static inline void clear_restore_sigmask(void)
+{
+ clear_thread_flag(TIF_RESTORE_SIGMASK);
+}
+static inline bool test_restore_sigmask(void)
+{
+ return test_thread_flag(TIF_RESTORE_SIGMASK);
+}
+static inline bool test_and_clear_restore_sigmask(void)
+{
+ return test_and_clear_thread_flag(TIF_RESTORE_SIGMASK);
+}
+
+#else /* TIF_RESTORE_SIGMASK */
+
+/* Higher-quality implementation, used if TIF_RESTORE_SIGMASK doesn't exist. */
+static inline void set_restore_sigmask(void)
+{
+ current->restore_sigmask = true;
+ WARN_ON(!test_thread_flag(TIF_SIGPENDING));
+}
+static inline void clear_restore_sigmask(void)
+{
+ current->restore_sigmask = false;
+}
+static inline bool test_restore_sigmask(void)
+{
+ return current->restore_sigmask;
+}
+static inline bool test_and_clear_restore_sigmask(void)
+{
+ if (!current->restore_sigmask)
+ return false;
+ current->restore_sigmask = false;
+ return true;
+}
+#endif
+
+static inline void restore_saved_sigmask(void)
+{
+ if (test_and_clear_restore_sigmask())
+ __set_current_blocked(¤t->saved_sigmask);
+}
+
+static inline sigset_t *sigmask_to_save(void)
+{
+ sigset_t *res = ¤t->blocked;
+ if (unlikely(test_restore_sigmask()))
+ res = ¤t->saved_sigmask;
+ return res;
+}
+
+static inline int kill_cad_pid(int sig, int priv)
+{
+ return kill_pid(cad_pid, sig, priv);
+}
+
+/* These can be the second arg to send_sig_info/send_group_sig_info. */
+#define SEND_SIG_NOINFO ((struct siginfo *) 0)
+#define SEND_SIG_PRIV ((struct siginfo *) 1)
+#define SEND_SIG_FORCED ((struct siginfo *) 2)
+
+/*
+ * True if we are on the alternate signal stack.
+ */
+static inline int on_sig_stack(unsigned long sp)
+{
+ /*
+ * If the signal stack is SS_AUTODISARM then, by construction, we
+ * can't be on the signal stack unless user code deliberately set
+ * SS_AUTODISARM when we were already on it.
+ *
+ * This improves reliability: if user state gets corrupted such that
+ * the stack pointer points very close to the end of the signal stack,
+ * then this check will enable the signal to be handled anyway.
+ */
+ if (current->sas_ss_flags & SS_AUTODISARM)
+ return 0;
+
+#ifdef CONFIG_STACK_GROWSUP
+ return sp >= current->sas_ss_sp &&
+ sp - current->sas_ss_sp < current->sas_ss_size;
+#else
+ return sp > current->sas_ss_sp &&
+ sp - current->sas_ss_sp <= current->sas_ss_size;
+#endif
+}
+
+static inline int sas_ss_flags(unsigned long sp)
+{
+ if (!current->sas_ss_size)
+ return SS_DISABLE;
+
+ return on_sig_stack(sp) ? SS_ONSTACK : 0;
+}
+
+static inline void sas_ss_reset(struct task_struct *p)
+{
+ p->sas_ss_sp = 0;
+ p->sas_ss_size = 0;
+ p->sas_ss_flags = SS_DISABLE;
+}
+
+static inline unsigned long sigsp(unsigned long sp, struct ksignal *ksig)
+{
+ if (unlikely((ksig->ka.sa.sa_flags & SA_ONSTACK)) && ! sas_ss_flags(sp))
+#ifdef CONFIG_STACK_GROWSUP
+ return current->sas_ss_sp;
+#else
+ return current->sas_ss_sp + current->sas_ss_size;
+#endif
+ return sp;
+}
+
+extern void __cleanup_sighand(struct sighand_struct *);
+extern void flush_itimer_signals(void);
+
+#define tasklist_empty() \
+ list_empty(&init_task.tasks)
+
+#define next_task(p) \
+ list_entry_rcu((p)->tasks.next, struct task_struct, tasks)
+
+#define for_each_process(p) \
+ for (p = &init_task ; (p = next_task(p)) != &init_task ; )
+
+extern bool current_is_single_threaded(void);
+
+/*
+ * Careful: do_each_thread/while_each_thread is a double loop so
+ * 'break' will not work as expected - use goto instead.
+ */
+#define do_each_thread(g, t) \
+ for (g = t = &init_task ; (g = t = next_task(g)) != &init_task ; ) do
+
+#define while_each_thread(g, t) \
+ while ((t = next_thread(t)) != g)
+
+#define __for_each_thread(signal, t) \
+ list_for_each_entry_rcu(t, &(signal)->thread_head, thread_node)
+
+#define for_each_thread(p, t) \
+ __for_each_thread((p)->signal, t)
+
+/* Careful: this is a double loop, 'break' won't work as expected. */
+#define for_each_process_thread(p, t) \
+ for_each_process(p) for_each_thread(p, t)
+
+typedef int (*proc_visitor)(struct task_struct *p, void *data);
+void walk_process_tree(struct task_struct *top, proc_visitor, void *);
+
+static inline int get_nr_threads(struct task_struct *tsk)
+{
+ return tsk->signal->nr_threads;
+}
+
+static inline bool thread_group_leader(struct task_struct *p)
+{
+ return p->exit_signal >= 0;
+}
+
+/* Do to the insanities of de_thread it is possible for a process
+ * to have the pid of the thread group leader without actually being
+ * the thread group leader. For iteration through the pids in proc
+ * all we care about is that we have a task with the appropriate
+ * pid, we don't actually care if we have the right task.
+ */
+static inline bool has_group_leader_pid(struct task_struct *p)
+{
+ return task_pid(p) == p->signal->leader_pid;
+}
+
+static inline
+bool same_thread_group(struct task_struct *p1, struct task_struct *p2)
+{
+ return p1->signal == p2->signal;
+}
+
+static inline struct task_struct *next_thread(const struct task_struct *p)
+{
+ return list_entry_rcu(p->thread_group.next,
+ struct task_struct, thread_group);
+}
+
+static inline int thread_group_empty(struct task_struct *p)
+{
+ return list_empty(&p->thread_group);
+}
+
+#define delay_group_leader(p) \
+ (thread_group_leader(p) && !thread_group_empty(p))
+
+extern struct sighand_struct *__lock_task_sighand(struct task_struct *tsk,
+ unsigned long *flags);
+
+static inline struct sighand_struct *lock_task_sighand(struct task_struct *tsk,
+ unsigned long *flags)
+{
+ struct sighand_struct *ret;
+
+ ret = __lock_task_sighand(tsk, flags);
+ (void)__cond_lock(&tsk->sighand->siglock, ret);
+ return ret;
+}
+
+static inline void unlock_task_sighand(struct task_struct *tsk,
+ unsigned long *flags)
+{
+ spin_unlock_irqrestore(&tsk->sighand->siglock, *flags);
+}
+
+static inline unsigned long task_rlimit(const struct task_struct *tsk,
+ unsigned int limit)
+{
+ return READ_ONCE(tsk->signal->rlim[limit].rlim_cur);
+}
+
+static inline unsigned long task_rlimit_max(const struct task_struct *tsk,
+ unsigned int limit)
+{
+ return READ_ONCE(tsk->signal->rlim[limit].rlim_max);
+}
+
+static inline unsigned long rlimit(unsigned int limit)
+{
+ return task_rlimit(current, limit);
+}
+
+static inline unsigned long rlimit_max(unsigned int limit)
+{
+ return task_rlimit_max(current, limit);
+}
+
+#endif /* _LINUX_SCHED_SIGNAL_H */
--- /dev/null
+#ifndef _LINUX_SCHED_STAT_H
+#define _LINUX_SCHED_STAT_H
+
+#include <linux/percpu.h>
+
+/*
+ * Various counters maintained by the scheduler and fork(),
+ * exposed via /proc, sys.c or used by drivers via these APIs.
+ *
+ * ( Note that all these values are aquired without locking,
+ * so they can only be relied on in narrow circumstances. )
+ */
+
+extern unsigned long total_forks;
+extern int nr_threads;
+DECLARE_PER_CPU(unsigned long, process_counts);
+extern int nr_processes(void);
+extern unsigned long nr_running(void);
+extern bool single_task_running(void);
+extern unsigned long nr_iowait(void);
+extern unsigned long nr_iowait_cpu(int cpu);
+extern void get_iowait_load(unsigned long *nr_waiters, unsigned long *load);
+
+static inline int sched_info_on(void)
+{
+#ifdef CONFIG_SCHEDSTATS
+ return 1;
+#elif defined(CONFIG_TASK_DELAY_ACCT)
+ extern int delayacct_on;
+ return delayacct_on;
+#else
+ return 0;
+#endif
+}
+
+#ifdef CONFIG_SCHEDSTATS
+void force_schedstat_enabled(void);
+#endif
+
+#endif /* _LINUX_SCHED_STAT_H */
-#ifndef _SCHED_SYSCTL_H
-#define _SCHED_SYSCTL_H
+#ifndef _LINUX_SCHED_SYSCTL_H
+#define _LINUX_SCHED_SYSCTL_H
+
+#include <linux/types.h>
+
+struct ctl_table;
#ifdef CONFIG_DETECT_HUNG_TASK
extern int sysctl_hung_task_check_count;
void __user *buffer, size_t *lenp,
loff_t *ppos);
-#endif /* _SCHED_SYSCTL_H */
+#endif /* _LINUX_SCHED_SYSCTL_H */
--- /dev/null
+#ifndef _LINUX_SCHED_TASK_H
+#define _LINUX_SCHED_TASK_H
+
+/*
+ * Interface between the scheduler and various task lifetime (fork()/exit())
+ * functionality:
+ */
+
+#include <linux/sched.h>
+
+struct task_struct;
+union thread_union;
+
+/*
+ * This serializes "schedule()" and also protects
+ * the run-queue from deletions/modifications (but
+ * _adding_ to the beginning of the run-queue has
+ * a separate lock).
+ */
+extern rwlock_t tasklist_lock;
+extern spinlock_t mmlist_lock;
+
+extern union thread_union init_thread_union;
+extern struct task_struct init_task;
+
+#ifdef CONFIG_PROVE_RCU
+extern int lockdep_tasklist_lock_is_held(void);
+#endif /* #ifdef CONFIG_PROVE_RCU */
+
+extern asmlinkage void schedule_tail(struct task_struct *prev);
+extern void init_idle(struct task_struct *idle, int cpu);
+extern void init_idle_bootup_task(struct task_struct *idle);
+
+extern int sched_fork(unsigned long clone_flags, struct task_struct *p);
+extern void sched_dead(struct task_struct *p);
+
+void __noreturn do_task_dead(void);
+
+extern void proc_caches_init(void);
+
+extern void release_task(struct task_struct * p);
+
+#ifdef CONFIG_HAVE_COPY_THREAD_TLS
+extern int copy_thread_tls(unsigned long, unsigned long, unsigned long,
+ struct task_struct *, unsigned long);
+#else
+extern int copy_thread(unsigned long, unsigned long, unsigned long,
+ struct task_struct *);
+
+/* Architectures that haven't opted into copy_thread_tls get the tls argument
+ * via pt_regs, so ignore the tls argument passed via C. */
+static inline int copy_thread_tls(
+ unsigned long clone_flags, unsigned long sp, unsigned long arg,
+ struct task_struct *p, unsigned long tls)
+{
+ return copy_thread(clone_flags, sp, arg, p);
+}
+#endif
+extern void flush_thread(void);
+
+#ifdef CONFIG_HAVE_EXIT_THREAD
+extern void exit_thread(struct task_struct *tsk);
+#else
+static inline void exit_thread(struct task_struct *tsk)
+{
+}
+#endif
+extern void do_group_exit(int);
+
+extern void exit_files(struct task_struct *);
+extern void exit_itimers(struct signal_struct *);
+
+extern long _do_fork(unsigned long, unsigned long, unsigned long, int __user *, int __user *, unsigned long);
+extern long do_fork(unsigned long, unsigned long, unsigned long, int __user *, int __user *);
+struct task_struct *fork_idle(int);
+extern pid_t kernel_thread(int (*fn)(void *), void *arg, unsigned long flags);
+
+extern void free_task(struct task_struct *tsk);
+
+/* sched_exec is called by processes performing an exec */
+#ifdef CONFIG_SMP
+extern void sched_exec(void);
+#else
+#define sched_exec() {}
+#endif
+
+#define get_task_struct(tsk) do { atomic_inc(&(tsk)->usage); } while(0)
+
+extern void __put_task_struct(struct task_struct *t);
+
+static inline void put_task_struct(struct task_struct *t)
+{
+ if (atomic_dec_and_test(&t->usage))
+ __put_task_struct(t);
+}
+
+struct task_struct *task_rcu_dereference(struct task_struct **ptask);
+struct task_struct *try_get_task_struct(struct task_struct **ptask);
+
+
+#ifdef CONFIG_ARCH_WANTS_DYNAMIC_TASK_STRUCT
+extern int arch_task_struct_size __read_mostly;
+#else
+# define arch_task_struct_size (sizeof(struct task_struct))
+#endif
+
+#ifdef CONFIG_VMAP_STACK
+static inline struct vm_struct *task_stack_vm_area(const struct task_struct *t)
+{
+ return t->stack_vm_area;
+}
+#else
+static inline struct vm_struct *task_stack_vm_area(const struct task_struct *t)
+{
+ return NULL;
+}
+#endif
+
+/*
+ * Protects ->fs, ->files, ->mm, ->group_info, ->comm, keyring
+ * subscriptions and synchronises with wait4(). Also used in procfs. Also
+ * pins the final release of task.io_context. Also protects ->cpuset and
+ * ->cgroup.subsys[]. And ->vfork_done.
+ *
+ * Nests both inside and outside of read_lock(&tasklist_lock).
+ * It must not be nested with write_lock_irq(&tasklist_lock),
+ * neither inside nor outside.
+ */
+static inline void task_lock(struct task_struct *p)
+{
+ spin_lock(&p->alloc_lock);
+}
+
+static inline void task_unlock(struct task_struct *p)
+{
+ spin_unlock(&p->alloc_lock);
+}
+
+#endif /* _LINUX_SCHED_TASK_H */
--- /dev/null
+#ifndef _LINUX_SCHED_TASK_STACK_H
+#define _LINUX_SCHED_TASK_STACK_H
+
+/*
+ * task->stack (kernel stack) handling interfaces:
+ */
+
+#include <linux/sched.h>
+#include <linux/magic.h>
+
+#ifdef CONFIG_THREAD_INFO_IN_TASK
+
+/*
+ * When accessing the stack of a non-current task that might exit, use
+ * try_get_task_stack() instead. task_stack_page will return a pointer
+ * that could get freed out from under you.
+ */
+static inline void *task_stack_page(const struct task_struct *task)
+{
+ return task->stack;
+}
+
+#define setup_thread_stack(new,old) do { } while(0)
+
+static inline unsigned long *end_of_stack(const struct task_struct *task)
+{
+ return task->stack;
+}
+
+#elif !defined(__HAVE_THREAD_FUNCTIONS)
+
+#define task_stack_page(task) ((void *)(task)->stack)
+
+static inline void setup_thread_stack(struct task_struct *p, struct task_struct *org)
+{
+ *task_thread_info(p) = *task_thread_info(org);
+ task_thread_info(p)->task = p;
+}
+
+/*
+ * Return the address of the last usable long on the stack.
+ *
+ * When the stack grows down, this is just above the thread
+ * info struct. Going any lower will corrupt the threadinfo.
+ *
+ * When the stack grows up, this is the highest address.
+ * Beyond that position, we corrupt data on the next page.
+ */
+static inline unsigned long *end_of_stack(struct task_struct *p)
+{
+#ifdef CONFIG_STACK_GROWSUP
+ return (unsigned long *)((unsigned long)task_thread_info(p) + THREAD_SIZE) - 1;
+#else
+ return (unsigned long *)(task_thread_info(p) + 1);
+#endif
+}
+
+#endif
+
+#ifdef CONFIG_THREAD_INFO_IN_TASK
+static inline void *try_get_task_stack(struct task_struct *tsk)
+{
+ return atomic_inc_not_zero(&tsk->stack_refcount) ?
+ task_stack_page(tsk) : NULL;
+}
+
+extern void put_task_stack(struct task_struct *tsk);
+#else
+static inline void *try_get_task_stack(struct task_struct *tsk)
+{
+ return task_stack_page(tsk);
+}
+
+static inline void put_task_stack(struct task_struct *tsk) {}
+#endif
+
+#define task_stack_end_corrupted(task) \
+ (*(end_of_stack(task)) != STACK_END_MAGIC)
+
+static inline int object_is_on_stack(void *obj)
+{
+ void *stack = task_stack_page(current);
+
+ return (obj >= stack) && (obj < (stack + THREAD_SIZE));
+}
+
+extern void thread_stack_cache_init(void);
+
+#ifdef CONFIG_DEBUG_STACK_USAGE
+static inline unsigned long stack_not_used(struct task_struct *p)
+{
+ unsigned long *n = end_of_stack(p);
+
+ do { /* Skip over canary */
+# ifdef CONFIG_STACK_GROWSUP
+ n--;
+# else
+ n++;
+# endif
+ } while (!*n);
+
+# ifdef CONFIG_STACK_GROWSUP
+ return (unsigned long)end_of_stack(p) - (unsigned long)n;
+# else
+ return (unsigned long)n - (unsigned long)end_of_stack(p);
+# endif
+}
+#endif
+extern void set_task_stack_end_magic(struct task_struct *tsk);
+
+#ifndef __HAVE_ARCH_KSTACK_END
+static inline int kstack_end(void *addr)
+{
+ /* Reliable end of stack detection:
+ * Some APM bios versions misalign the stack
+ */
+ return !(((unsigned long)addr+sizeof(void*)-1) & (THREAD_SIZE-sizeof(void*)));
+}
+#endif
+
+#endif /* _LINUX_SCHED_TASK_STACK_H */
--- /dev/null
+#ifndef _LINUX_SCHED_TOPOLOGY_H
+#define _LINUX_SCHED_TOPOLOGY_H
+
+#include <linux/topology.h>
+
+#include <linux/sched/idle.h>
+
+/*
+ * sched-domains (multiprocessor balancing) declarations:
+ */
+#ifdef CONFIG_SMP
+
+#define SD_LOAD_BALANCE 0x0001 /* Do load balancing on this domain. */
+#define SD_BALANCE_NEWIDLE 0x0002 /* Balance when about to become idle */
+#define SD_BALANCE_EXEC 0x0004 /* Balance on exec */
+#define SD_BALANCE_FORK 0x0008 /* Balance on fork, clone */
+#define SD_BALANCE_WAKE 0x0010 /* Balance on wakeup */
+#define SD_WAKE_AFFINE 0x0020 /* Wake task to waking CPU */
+#define SD_ASYM_CPUCAPACITY 0x0040 /* Groups have different max cpu capacities */
+#define SD_SHARE_CPUCAPACITY 0x0080 /* Domain members share cpu capacity */
+#define SD_SHARE_POWERDOMAIN 0x0100 /* Domain members share power domain */
+#define SD_SHARE_PKG_RESOURCES 0x0200 /* Domain members share cpu pkg resources */
+#define SD_SERIALIZE 0x0400 /* Only a single load balancing instance */
+#define SD_ASYM_PACKING 0x0800 /* Place busy groups earlier in the domain */
+#define SD_PREFER_SIBLING 0x1000 /* Prefer to place tasks in a sibling domain */
+#define SD_OVERLAP 0x2000 /* sched_domains of this level overlap */
+#define SD_NUMA 0x4000 /* cross-node balancing */
+
+/*
+ * Increase resolution of cpu_capacity calculations
+ */
+#define SCHED_CAPACITY_SHIFT SCHED_FIXEDPOINT_SHIFT
+#define SCHED_CAPACITY_SCALE (1L << SCHED_CAPACITY_SHIFT)
+
+#ifdef CONFIG_SCHED_SMT
+static inline int cpu_smt_flags(void)
+{
+ return SD_SHARE_CPUCAPACITY | SD_SHARE_PKG_RESOURCES;
+}
+#endif
+
+#ifdef CONFIG_SCHED_MC
+static inline int cpu_core_flags(void)
+{
+ return SD_SHARE_PKG_RESOURCES;
+}
+#endif
+
+#ifdef CONFIG_NUMA
+static inline int cpu_numa_flags(void)
+{
+ return SD_NUMA;
+}
+#endif
+
+extern int arch_asym_cpu_priority(int cpu);
+
+struct sched_domain_attr {
+ int relax_domain_level;
+};
+
+#define SD_ATTR_INIT (struct sched_domain_attr) { \
+ .relax_domain_level = -1, \
+}
+
+extern int sched_domain_level_max;
+
+struct sched_group;
+
+struct sched_domain_shared {
+ atomic_t ref;
+ atomic_t nr_busy_cpus;
+ int has_idle_cores;
+};
+
+struct sched_domain {
+ /* These fields must be setup */
+ struct sched_domain *parent; /* top domain must be null terminated */
+ struct sched_domain *child; /* bottom domain must be null terminated */
+ struct sched_group *groups; /* the balancing groups of the domain */
+ unsigned long min_interval; /* Minimum balance interval ms */
+ unsigned long max_interval; /* Maximum balance interval ms */
+ unsigned int busy_factor; /* less balancing by factor if busy */
+ unsigned int imbalance_pct; /* No balance until over watermark */
+ unsigned int cache_nice_tries; /* Leave cache hot tasks for # tries */
+ unsigned int busy_idx;
+ unsigned int idle_idx;
+ unsigned int newidle_idx;
+ unsigned int wake_idx;
+ unsigned int forkexec_idx;
+ unsigned int smt_gain;
+
+ int nohz_idle; /* NOHZ IDLE status */
+ int flags; /* See SD_* */
+ int level;
+
+ /* Runtime fields. */
+ unsigned long last_balance; /* init to jiffies. units in jiffies */
+ unsigned int balance_interval; /* initialise to 1. units in ms. */
+ unsigned int nr_balance_failed; /* initialise to 0 */
+
+ /* idle_balance() stats */
+ u64 max_newidle_lb_cost;
+ unsigned long next_decay_max_lb_cost;
+
+ u64 avg_scan_cost; /* select_idle_sibling */
+
+#ifdef CONFIG_SCHEDSTATS
+ /* load_balance() stats */
+ unsigned int lb_count[CPU_MAX_IDLE_TYPES];
+ unsigned int lb_failed[CPU_MAX_IDLE_TYPES];
+ unsigned int lb_balanced[CPU_MAX_IDLE_TYPES];
+ unsigned int lb_imbalance[CPU_MAX_IDLE_TYPES];
+ unsigned int lb_gained[CPU_MAX_IDLE_TYPES];
+ unsigned int lb_hot_gained[CPU_MAX_IDLE_TYPES];
+ unsigned int lb_nobusyg[CPU_MAX_IDLE_TYPES];
+ unsigned int lb_nobusyq[CPU_MAX_IDLE_TYPES];
+
+ /* Active load balancing */
+ unsigned int alb_count;
+ unsigned int alb_failed;
+ unsigned int alb_pushed;
+
+ /* SD_BALANCE_EXEC stats */
+ unsigned int sbe_count;
+ unsigned int sbe_balanced;
+ unsigned int sbe_pushed;
+
+ /* SD_BALANCE_FORK stats */
+ unsigned int sbf_count;
+ unsigned int sbf_balanced;
+ unsigned int sbf_pushed;
+
+ /* try_to_wake_up() stats */
+ unsigned int ttwu_wake_remote;
+ unsigned int ttwu_move_affine;
+ unsigned int ttwu_move_balance;
+#endif
+#ifdef CONFIG_SCHED_DEBUG
+ char *name;
+#endif
+ union {
+ void *private; /* used during construction */
+ struct rcu_head rcu; /* used during destruction */
+ };
+ struct sched_domain_shared *shared;
+
+ unsigned int span_weight;
+ /*
+ * Span of all CPUs in this domain.
+ *
+ * NOTE: this field is variable length. (Allocated dynamically
+ * by attaching extra space to the end of the structure,
+ * depending on how many CPUs the kernel has booted up with)
+ */
+ unsigned long span[0];
+};
+
+static inline struct cpumask *sched_domain_span(struct sched_domain *sd)
+{
+ return to_cpumask(sd->span);
+}
+
+extern void partition_sched_domains(int ndoms_new, cpumask_var_t doms_new[],
+ struct sched_domain_attr *dattr_new);
+
+/* Allocate an array of sched domains, for partition_sched_domains(). */
+cpumask_var_t *alloc_sched_domains(unsigned int ndoms);
+void free_sched_domains(cpumask_var_t doms[], unsigned int ndoms);
+
+bool cpus_share_cache(int this_cpu, int that_cpu);
+
+typedef const struct cpumask *(*sched_domain_mask_f)(int cpu);
+typedef int (*sched_domain_flags_f)(void);
+
+#define SDTL_OVERLAP 0x01
+
+struct sd_data {
+ struct sched_domain **__percpu sd;
+ struct sched_domain_shared **__percpu sds;
+ struct sched_group **__percpu sg;
+ struct sched_group_capacity **__percpu sgc;
+};
+
+struct sched_domain_topology_level {
+ sched_domain_mask_f mask;
+ sched_domain_flags_f sd_flags;
+ int flags;
+ int numa_level;
+ struct sd_data data;
+#ifdef CONFIG_SCHED_DEBUG
+ char *name;
+#endif
+};
+
+extern void set_sched_topology(struct sched_domain_topology_level *tl);
+
+#ifdef CONFIG_SCHED_DEBUG
+# define SD_INIT_NAME(type) .name = #type
+#else
+# define SD_INIT_NAME(type)
+#endif
+
+#else /* CONFIG_SMP */
+
+struct sched_domain_attr;
+
+static inline void
+partition_sched_domains(int ndoms_new, cpumask_var_t doms_new[],
+ struct sched_domain_attr *dattr_new)
+{
+}
+
+static inline bool cpus_share_cache(int this_cpu, int that_cpu)
+{
+ return true;
+}
+
+#endif /* !CONFIG_SMP */
+
+static inline int task_node(const struct task_struct *p)
+{
+ return cpu_to_node(task_cpu(p));
+}
+
+#endif /* _LINUX_SCHED_TOPOLOGY_H */
--- /dev/null
+#ifndef _LINUX_SCHED_USER_H
+#define _LINUX_SCHED_USER_H
+
+#include <linux/uidgid.h>
+#include <linux/atomic.h>
+
+struct key;
+
+/*
+ * Some day this will be a full-fledged user tracking system..
+ */
+struct user_struct {
+ atomic_t __count; /* reference count */
+ atomic_t processes; /* How many processes does this user have? */
+ atomic_t sigpending; /* How many pending signals does this user have? */
+#ifdef CONFIG_FANOTIFY
+ atomic_t fanotify_listeners;
+#endif
+#ifdef CONFIG_EPOLL
+ atomic_long_t epoll_watches; /* The number of file descriptors currently watched */
+#endif
+#ifdef CONFIG_POSIX_MQUEUE
+ /* protected by mq_lock */
+ unsigned long mq_bytes; /* How many bytes can be allocated to mqueue? */
+#endif
+ unsigned long locked_shm; /* How many pages of mlocked shm ? */
+ unsigned long unix_inflight; /* How many files in flight in unix sockets */
+ atomic_long_t pipe_bufs; /* how many pages are allocated in pipe buffers */
+
+#ifdef CONFIG_KEYS
+ struct key *uid_keyring; /* UID specific keyring */
+ struct key *session_keyring; /* UID's default session keyring */
+#endif
+
+ /* Hash table maintenance information */
+ struct hlist_node uidhash_node;
+ kuid_t uid;
+
+#if defined(CONFIG_PERF_EVENTS) || defined(CONFIG_BPF_SYSCALL)
+ atomic_long_t locked_vm;
+#endif
+};
+
+extern int uids_sysfs_init(void);
+
+extern struct user_struct *find_user(kuid_t);
+
+extern struct user_struct root_user;
+#define INIT_USER (&root_user)
+
+
+/* per-UID process charging. */
+extern struct user_struct * alloc_uid(kuid_t);
+static inline struct user_struct *get_uid(struct user_struct *u)
+{
+ atomic_inc(&u->__count);
+ return u;
+}
+extern void free_uid(struct user_struct *);
+
+#endif /* _LINUX_SCHED_USER_H */
--- /dev/null
+#ifndef _LINUX_SCHED_WAKE_Q_H
+#define _LINUX_SCHED_WAKE_Q_H
+
+/*
+ * Wake-queues are lists of tasks with a pending wakeup, whose
+ * callers have already marked the task as woken internally,
+ * and can thus carry on. A common use case is being able to
+ * do the wakeups once the corresponding user lock as been
+ * released.
+ *
+ * We hold reference to each task in the list across the wakeup,
+ * thus guaranteeing that the memory is still valid by the time
+ * the actual wakeups are performed in wake_up_q().
+ *
+ * One per task suffices, because there's never a need for a task to be
+ * in two wake queues simultaneously; it is forbidden to abandon a task
+ * in a wake queue (a call to wake_up_q() _must_ follow), so if a task is
+ * already in a wake queue, the wakeup will happen soon and the second
+ * waker can just skip it.
+ *
+ * The DEFINE_WAKE_Q macro declares and initializes the list head.
+ * wake_up_q() does NOT reinitialize the list; it's expected to be
+ * called near the end of a function. Otherwise, the list can be
+ * re-initialized for later re-use by wake_q_init().
+ *
+ * Note that this can cause spurious wakeups. schedule() callers
+ * must ensure the call is done inside a loop, confirming that the
+ * wakeup condition has in fact occurred.
+ */
+
+#include <linux/sched.h>
+
+struct wake_q_head {
+ struct wake_q_node *first;
+ struct wake_q_node **lastp;
+};
+
+#define WAKE_Q_TAIL ((struct wake_q_node *) 0x01)
+
+#define DEFINE_WAKE_Q(name) \
+ struct wake_q_head name = { WAKE_Q_TAIL, &name.first }
+
+static inline void wake_q_init(struct wake_q_head *head)
+{
+ head->first = WAKE_Q_TAIL;
+ head->lastp = &head->first;
+}
+
+extern void wake_q_add(struct wake_q_head *head,
+ struct task_struct *task);
+extern void wake_up_q(struct wake_q_head *head);
+
+#endif /* _LINUX_SCHED_WAKE_Q_H */
--- /dev/null
+#ifndef _LINUX_SCHED_XACCT_H
+#define _LINUX_SCHED_XACCT_H
+
+/*
+ * Extended task accounting methods:
+ */
+
+#include <linux/sched.h>
+
+#ifdef CONFIG_TASK_XACCT
+static inline void add_rchar(struct task_struct *tsk, ssize_t amt)
+{
+ tsk->ioac.rchar += amt;
+}
+
+static inline void add_wchar(struct task_struct *tsk, ssize_t amt)
+{
+ tsk->ioac.wchar += amt;
+}
+
+static inline void inc_syscr(struct task_struct *tsk)
+{
+ tsk->ioac.syscr++;
+}
+
+static inline void inc_syscw(struct task_struct *tsk)
+{
+ tsk->ioac.syscw++;
+}
+#else
+static inline void add_rchar(struct task_struct *tsk, ssize_t amt)
+{
+}
+
+static inline void add_wchar(struct task_struct *tsk, ssize_t amt)
+{
+}
+
+static inline void inc_syscr(struct task_struct *tsk)
+{
+}
+
+static inline void inc_syscw(struct task_struct *tsk)
+{
+}
+#endif
+
+#endif /* _LINUX_SCHED_XACCT_H */
#ifndef _LINUX_SIGNAL_H
#define _LINUX_SIGNAL_H
-#include <linux/list.h>
#include <linux/bug.h>
-#include <uapi/linux/signal.h>
+#include <linux/signal_types.h>
struct task_struct;
/* for sysctl */
extern int print_fatal_signals;
-/*
- * Real Time signals may be queued.
- */
-
-struct sigqueue {
- struct list_head list;
- int flags;
- siginfo_t info;
- struct user_struct *user;
-};
-
-/* flags values. */
-#define SIGQUEUE_PREALLOC 1
-
-struct sigpending {
- struct list_head list;
- sigset_t signal;
-};
#ifndef HAVE_ARCH_COPY_SIGINFO
extern void __set_current_blocked(const sigset_t *);
extern int show_unhandled_signals;
-struct sigaction {
-#ifndef __ARCH_HAS_IRIX_SIGACTION
- __sighandler_t sa_handler;
- unsigned long sa_flags;
-#else
- unsigned int sa_flags;
- __sighandler_t sa_handler;
-#endif
-#ifdef __ARCH_HAS_SA_RESTORER
- __sigrestore_t sa_restorer;
-#endif
- sigset_t sa_mask; /* mask last for extensibility */
-};
-
-struct k_sigaction {
- struct sigaction sa;
-#ifdef __ARCH_HAS_KA_RESTORER
- __sigrestore_t ka_restorer;
-#endif
-};
-
-#ifdef CONFIG_OLD_SIGACTION
-struct old_sigaction {
- __sighandler_t sa_handler;
- old_sigset_t sa_mask;
- unsigned long sa_flags;
- __sigrestore_t sa_restorer;
-};
-#endif
-
-struct ksignal {
- struct k_sigaction ka;
- siginfo_t info;
- int sig;
-};
-
extern int get_signal(struct ksignal *ksig);
extern void signal_setup_done(int failed, struct ksignal *ksig, int stepping);
extern void exit_signals(struct task_struct *tsk);
--- /dev/null
+#ifndef _LINUX_SIGNAL_TYPES_H
+#define _LINUX_SIGNAL_TYPES_H
+
+/*
+ * Basic signal handling related data type definitions:
+ */
+
+#include <linux/list.h>
+#include <uapi/linux/signal.h>
+
+/*
+ * Real Time signals may be queued.
+ */
+
+struct sigqueue {
+ struct list_head list;
+ int flags;
+ siginfo_t info;
+ struct user_struct *user;
+};
+
+/* flags values. */
+#define SIGQUEUE_PREALLOC 1
+
+struct sigpending {
+ struct list_head list;
+ sigset_t signal;
+};
+
+struct sigaction {
+#ifndef __ARCH_HAS_IRIX_SIGACTION
+ __sighandler_t sa_handler;
+ unsigned long sa_flags;
+#else
+ unsigned int sa_flags;
+ __sighandler_t sa_handler;
+#endif
+#ifdef __ARCH_HAS_SA_RESTORER
+ __sigrestore_t sa_restorer;
+#endif
+ sigset_t sa_mask; /* mask last for extensibility */
+};
+
+struct k_sigaction {
+ struct sigaction sa;
+#ifdef __ARCH_HAS_KA_RESTORER
+ __sigrestore_t ka_restorer;
+#endif
+};
+
+#ifdef CONFIG_OLD_SIGACTION
+struct old_sigaction {
+ __sighandler_t sa_handler;
+ old_sigset_t sa_mask;
+ unsigned long sa_flags;
+ __sigrestore_t sa_restorer;
+};
+#endif
+
+struct ksignal {
+ struct k_sigaction ka;
+ siginfo_t info;
+ int sig;
+};
+
+#endif /* _LINUX_SIGNAL_TYPES_H */
#define _LINUX_SIGNALFD_H
#include <uapi/linux/signalfd.h>
-
+#include <linux/sched/signal.h>
#ifdef CONFIG_SIGNALFD
#include <linux/dma-mapping.h>
#include <linux/netdev_features.h>
#include <linux/sched.h>
+#include <linux/sched/clock.h>
#include <net/flow_dissector.h>
#include <linux/splice.h>
#include <linux/in6.h>
#include <linux/time.h>
#include <linux/uidgid.h>
+#define KSTAT_QUERY_FLAGS (AT_STATX_SYNC_TYPE)
+
struct kstat {
- u64 ino;
- dev_t dev;
+ u32 result_mask; /* What fields the user got */
umode_t mode;
unsigned int nlink;
+ uint32_t blksize; /* Preferred I/O size */
+ u64 attributes;
+#define KSTAT_ATTR_FS_IOC_FLAGS \
+ (STATX_ATTR_COMPRESSED | \
+ STATX_ATTR_IMMUTABLE | \
+ STATX_ATTR_APPEND | \
+ STATX_ATTR_NODUMP | \
+ STATX_ATTR_ENCRYPTED \
+ )/* Attrs corresponding to FS_*_FL flags */
+ u64 ino;
+ dev_t dev;
+ dev_t rdev;
kuid_t uid;
kgid_t gid;
- dev_t rdev;
loff_t size;
- struct timespec atime;
+ struct timespec atime;
struct timespec mtime;
struct timespec ctime;
- unsigned long blksize;
- unsigned long long blocks;
+ struct timespec btime; /* File creation time */
+ u64 blocks;
};
#endif
#define _LINUX_SUNRPC_TYPES_H_
#include <linux/timer.h>
+#include <linux/sched/signal.h>
#include <linux/workqueue.h>
#include <linux/sunrpc/debug.h>
#include <linux/list.h>
struct stat64;
struct statfs;
struct statfs64;
+struct statx;
struct __sysctl_args;
struct sysinfo;
struct timespec;
unsigned long prot, int pkey);
asmlinkage long sys_pkey_alloc(unsigned long flags, unsigned long init_val);
asmlinkage long sys_pkey_free(int pkey);
+asmlinkage long sys_statx(int dfd, const char __user *path, unsigned flags,
+ unsigned mask, struct statx __user *buffer);
#endif
#define _LINUX_TASKSTATS_KERN_H
#include <linux/taskstats.h>
-#include <linux/sched.h>
+#include <linux/sched/signal.h>
#include <linux/slab.h>
#ifdef CONFIG_TASKSTATS
void timekeeping_init(void);
extern int timekeeping_suspended;
+/* Architecture timer tick functions: */
+extern void update_process_times(int user);
+extern void xtime_update(unsigned long ticks);
+
/*
* Get and set timeofday
*/
extern enum hrtimer_restart it_real_fn(struct hrtimer *);
#if defined(CONFIG_SMP) && defined(CONFIG_NO_HZ_COMMON)
-#include <linux/sysctl.h>
+struct ctl_table;
extern unsigned int sysctl_timer_migration;
int timer_migration_handler(struct ctl_table *table, int write,
#include <linux/nsproxy.h>
#include <linux/ns_common.h>
#include <linux/sched.h>
+#include <linux/workqueue.h>
+#include <linux/rwsem.h>
+#include <linux/sysctl.h>
#include <linux/err.h>
#define UID_GID_MAP_MAX_EXTENTS 5
struct hlist_node node;
struct user_namespace *ns;
kuid_t uid;
- atomic_t count;
+ int count;
atomic_t ucount[UCOUNT_COUNTS];
};
unsigned long from, unsigned long to,
unsigned long len);
-extern void userfaultfd_remove(struct vm_area_struct *vma,
- struct vm_area_struct **prev,
+extern bool userfaultfd_remove(struct vm_area_struct *vma,
unsigned long start,
unsigned long end);
extern void userfaultfd_unmap_complete(struct mm_struct *mm,
struct list_head *uf);
-extern void userfaultfd_exit(struct mm_struct *mm);
-
#else /* CONFIG_USERFAULTFD */
/* mm helpers */
{
}
-static inline void userfaultfd_remove(struct vm_area_struct *vma,
- struct vm_area_struct **prev,
+static inline bool userfaultfd_remove(struct vm_area_struct *vma,
unsigned long start,
unsigned long end)
{
+ return true;
}
static inline int userfaultfd_unmap_prep(struct vm_area_struct *vma,
{
}
-static inline void userfaultfd_exit(struct mm_struct *mm)
-{
-}
-
#endif /* CONFIG_USERFAULTFD */
#endif /* _LINUX_USERFAULTFD_K_H */
#include <linux/virtio_byteorder.h>
#include <uapi/linux/virtio_config.h>
+struct irq_affinity;
+
/**
* virtio_config_ops - operations for configuring a virtio device
* @get: read the value of a configuration field
* This returns a pointer to the bus name a la pci_name from which
* the caller can then copy.
* @set_vq_affinity: set the affinity for a virtqueue.
+ * @get_vq_affinity: get the affinity for a virtqueue (optional).
*/
typedef void vq_callback_t(struct virtqueue *);
struct virtio_config_ops {
void (*set_status)(struct virtio_device *vdev, u8 status);
void (*reset)(struct virtio_device *vdev);
int (*find_vqs)(struct virtio_device *, unsigned nvqs,
- struct virtqueue *vqs[],
- vq_callback_t *callbacks[],
- const char * const names[]);
+ struct virtqueue *vqs[], vq_callback_t *callbacks[],
+ const char * const names[], struct irq_affinity *desc);
void (*del_vqs)(struct virtio_device *);
u64 (*get_features)(struct virtio_device *vdev);
int (*finalize_features)(struct virtio_device *vdev);
const char *(*bus_name)(struct virtio_device *vdev);
int (*set_vq_affinity)(struct virtqueue *vq, int cpu);
+ const struct cpumask *(*get_vq_affinity)(struct virtio_device *vdev,
+ int index);
};
/* If driver didn't advertise the feature, it will never appear. */
vq_callback_t *callbacks[] = { c };
const char *names[] = { n };
struct virtqueue *vq;
- int err = vdev->config->find_vqs(vdev, 1, &vq, callbacks, names);
+ int err = vdev->config->find_vqs(vdev, 1, &vq, callbacks, names, NULL);
if (err < 0)
return ERR_PTR(err);
return vq;
+++ /dev/null
-/*
- * Virtio platform device driver
- *
- * Copyright 2011, ARM Ltd.
- *
- * Based on Virtio PCI driver by Anthony Liguori, copyright IBM Corp. 2007
- *
- * This header is BSD licensed so anyone can use the definitions to implement
- * compatible drivers/servers.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- * notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- * notice, this list of conditions and the following disclaimer in the
- * documentation and/or other materials provided with the distribution.
- * 3. Neither the name of IBM nor the names of its contributors
- * may be used to endorse or promote products derived from this software
- * without specific prior written permission.
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED. IN NO EVENT SHALL IBM OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- */
-
-#ifndef _LINUX_VIRTIO_MMIO_H
-#define _LINUX_VIRTIO_MMIO_H
-
-/*
- * Control registers
- */
-
-/* Magic value ("virt" string) - Read Only */
-#define VIRTIO_MMIO_MAGIC_VALUE 0x000
-
-/* Virtio device version - Read Only */
-#define VIRTIO_MMIO_VERSION 0x004
-
-/* Virtio device ID - Read Only */
-#define VIRTIO_MMIO_DEVICE_ID 0x008
-
-/* Virtio vendor ID - Read Only */
-#define VIRTIO_MMIO_VENDOR_ID 0x00c
-
-/* Bitmask of the features supported by the device (host)
- * (32 bits per set) - Read Only */
-#define VIRTIO_MMIO_DEVICE_FEATURES 0x010
-
-/* Device (host) features set selector - Write Only */
-#define VIRTIO_MMIO_DEVICE_FEATURES_SEL 0x014
-
-/* Bitmask of features activated by the driver (guest)
- * (32 bits per set) - Write Only */
-#define VIRTIO_MMIO_DRIVER_FEATURES 0x020
-
-/* Activated features set selector - Write Only */
-#define VIRTIO_MMIO_DRIVER_FEATURES_SEL 0x024
-
-
-#ifndef VIRTIO_MMIO_NO_LEGACY /* LEGACY DEVICES ONLY! */
-
-/* Guest's memory page size in bytes - Write Only */
-#define VIRTIO_MMIO_GUEST_PAGE_SIZE 0x028
-
-#endif
-
-
-/* Queue selector - Write Only */
-#define VIRTIO_MMIO_QUEUE_SEL 0x030
-
-/* Maximum size of the currently selected queue - Read Only */
-#define VIRTIO_MMIO_QUEUE_NUM_MAX 0x034
-
-/* Queue size for the currently selected queue - Write Only */
-#define VIRTIO_MMIO_QUEUE_NUM 0x038
-
-
-#ifndef VIRTIO_MMIO_NO_LEGACY /* LEGACY DEVICES ONLY! */
-
-/* Used Ring alignment for the currently selected queue - Write Only */
-#define VIRTIO_MMIO_QUEUE_ALIGN 0x03c
-
-/* Guest's PFN for the currently selected queue - Read Write */
-#define VIRTIO_MMIO_QUEUE_PFN 0x040
-
-#endif
-
-
-/* Ready bit for the currently selected queue - Read Write */
-#define VIRTIO_MMIO_QUEUE_READY 0x044
-
-/* Queue notifier - Write Only */
-#define VIRTIO_MMIO_QUEUE_NOTIFY 0x050
-
-/* Interrupt status - Read Only */
-#define VIRTIO_MMIO_INTERRUPT_STATUS 0x060
-
-/* Interrupt acknowledge - Write Only */
-#define VIRTIO_MMIO_INTERRUPT_ACK 0x064
-
-/* Device status register - Read Write */
-#define VIRTIO_MMIO_STATUS 0x070
-
-/* Selected queue's Descriptor Table address, 64 bits in two halves */
-#define VIRTIO_MMIO_QUEUE_DESC_LOW 0x080
-#define VIRTIO_MMIO_QUEUE_DESC_HIGH 0x084
-
-/* Selected queue's Available Ring address, 64 bits in two halves */
-#define VIRTIO_MMIO_QUEUE_AVAIL_LOW 0x090
-#define VIRTIO_MMIO_QUEUE_AVAIL_HIGH 0x094
-
-/* Selected queue's Used Ring address, 64 bits in two halves */
-#define VIRTIO_MMIO_QUEUE_USED_LOW 0x0a0
-#define VIRTIO_MMIO_QUEUE_USED_HIGH 0x0a4
-
-/* Configuration atomicity value */
-#define VIRTIO_MMIO_CONFIG_GENERATION 0x0fc
-
-/* The config space is defined by each driver as
- * the per-driver configuration space - Read Write */
-#define VIRTIO_MMIO_CONFIG 0x100
-
-
-
-/*
- * Interrupt flags (re: interrupt status & acknowledge registers)
- */
-
-#define VIRTIO_MMIO_INT_VRING (1 << 0)
-#define VIRTIO_MMIO_INT_CONFIG (1 << 1)
-
-#endif
THP_SPLIT_PAGE_FAILED,
THP_DEFERRED_SPLIT_PAGE,
THP_SPLIT_PMD,
+#ifdef CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD
+ THP_SPLIT_PUD,
+#endif
THP_ZERO_PAGE_ALLOC,
THP_ZERO_PAGE_ALLOC_FAILED,
#endif
static inline void vmacache_flush(struct task_struct *tsk)
{
- memset(tsk->vmacache, 0, sizeof(tsk->vmacache));
+ memset(tsk->vmacache.vmas, 0, sizeof(tsk->vmacache.vmas));
}
extern void vmacache_flush_all(struct mm_struct *mm);
#include <linux/list.h>
#include <linux/stddef.h>
#include <linux/spinlock.h>
+
#include <asm/current.h>
#include <uapi/linux/wait.h>
__ret; \
})
+extern int do_wait_intr(wait_queue_head_t *, wait_queue_t *);
+extern int do_wait_intr_irq(wait_queue_head_t *, wait_queue_t *);
-#define __wait_event_interruptible_locked(wq, condition, exclusive, irq) \
+#define __wait_event_interruptible_locked(wq, condition, exclusive, fn) \
({ \
- int __ret = 0; \
+ int __ret; \
DEFINE_WAIT(__wait); \
if (exclusive) \
__wait.flags |= WQ_FLAG_EXCLUSIVE; \
do { \
- if (likely(list_empty(&__wait.task_list))) \
- __add_wait_queue_tail(&(wq), &__wait); \
- set_current_state(TASK_INTERRUPTIBLE); \
- if (signal_pending(current)) { \
- __ret = -ERESTARTSYS; \
+ __ret = fn(&(wq), &__wait); \
+ if (__ret) \
break; \
- } \
- if (irq) \
- spin_unlock_irq(&(wq).lock); \
- else \
- spin_unlock(&(wq).lock); \
- schedule(); \
- if (irq) \
- spin_lock_irq(&(wq).lock); \
- else \
- spin_lock(&(wq).lock); \
} while (!(condition)); \
__remove_wait_queue(&(wq), &__wait); \
__set_current_state(TASK_RUNNING); \
*/
#define wait_event_interruptible_locked(wq, condition) \
((condition) \
- ? 0 : __wait_event_interruptible_locked(wq, condition, 0, 0))
+ ? 0 : __wait_event_interruptible_locked(wq, condition, 0, do_wait_intr))
/**
* wait_event_interruptible_locked_irq - sleep until a condition gets true
*/
#define wait_event_interruptible_locked_irq(wq, condition) \
((condition) \
- ? 0 : __wait_event_interruptible_locked(wq, condition, 0, 1))
+ ? 0 : __wait_event_interruptible_locked(wq, condition, 0, do_wait_intr_irq))
/**
* wait_event_interruptible_exclusive_locked - sleep exclusively until a condition gets true
*/
#define wait_event_interruptible_exclusive_locked(wq, condition) \
((condition) \
- ? 0 : __wait_event_interruptible_locked(wq, condition, 1, 0))
+ ? 0 : __wait_event_interruptible_locked(wq, condition, 1, do_wait_intr))
/**
* wait_event_interruptible_exclusive_locked_irq - sleep until a condition gets true
*/
#define wait_event_interruptible_exclusive_locked_irq(wq, condition) \
((condition) \
- ? 0 : __wait_event_interruptible_locked(wq, condition, 1, 1))
+ ? 0 : __wait_event_interruptible_locked(wq, condition, 1, do_wait_intr_irq))
#define __wait_event_killable(wq, condition) \
#include <linux/poll.h>
#include <linux/fs.h>
#include <linux/mutex.h>
+#include <linux/sched/signal.h>
#include <linux/compiler.h> /* need __user */
#include <linux/videodev2.h>
int vsp1_du_init(struct device *dev);
-int vsp1_du_setup_lif(struct device *dev, unsigned int width,
- unsigned int height);
+/**
+ * struct vsp1_du_lif_config - VSP LIF configuration
+ * @width: output frame width
+ * @height: output frame height
+ */
+struct vsp1_du_lif_config {
+ unsigned int width;
+ unsigned int height;
+};
+
+int vsp1_du_setup_lif(struct device *dev, const struct vsp1_du_lif_config *cfg);
struct vsp1_du_atomic_config {
u32 pixelformat;
u32 atime;
u32 mtime;
u64 length;
- char *name;
- char *uid;
- char *gid;
- char *muid;
+ const char *name;
+ const char *uid;
+ const char *gid;
+ const char *muid;
char *extension; /* 9p2000.u extensions */
kuid_t n_uid; /* 9p2000.u extensions */
kgid_t n_gid; /* 9p2000.u extensions */
void p9_client_disconnect(struct p9_client *clnt);
void p9_client_begin_disconnect(struct p9_client *clnt);
struct p9_fid *p9_client_attach(struct p9_client *clnt, struct p9_fid *afid,
- char *uname, kuid_t n_uname, char *aname);
+ const char *uname, kuid_t n_uname, const char *aname);
struct p9_fid *p9_client_walk(struct p9_fid *oldfid, uint16_t nwname,
- char **wnames, int clone);
+ const unsigned char * const *wnames, int clone);
int p9_client_open(struct p9_fid *fid, int mode);
-int p9_client_fcreate(struct p9_fid *fid, char *name, u32 perm, int mode,
+int p9_client_fcreate(struct p9_fid *fid, const char *name, u32 perm, int mode,
char *extension);
-int p9_client_link(struct p9_fid *fid, struct p9_fid *oldfid, char *newname);
-int p9_client_symlink(struct p9_fid *fid, char *name, char *symname, kgid_t gid,
- struct p9_qid *qid);
-int p9_client_create_dotl(struct p9_fid *ofid, char *name, u32 flags, u32 mode,
+int p9_client_link(struct p9_fid *fid, struct p9_fid *oldfid, const char *newname);
+int p9_client_symlink(struct p9_fid *fid, const char *name, const char *symname,
+ kgid_t gid, struct p9_qid *qid);
+int p9_client_create_dotl(struct p9_fid *ofid, const char *name, u32 flags, u32 mode,
kgid_t gid, struct p9_qid *qid);
int p9_client_clunk(struct p9_fid *fid);
int p9_client_fsync(struct p9_fid *fid, int datasync);
struct p9_stat_dotl *p9_client_getattr_dotl(struct p9_fid *fid,
u64 request_mask);
-int p9_client_mknod_dotl(struct p9_fid *oldfid, char *name, int mode,
+int p9_client_mknod_dotl(struct p9_fid *oldfid, const char *name, int mode,
dev_t rdev, kgid_t gid, struct p9_qid *);
-int p9_client_mkdir_dotl(struct p9_fid *fid, char *name, int mode,
+int p9_client_mkdir_dotl(struct p9_fid *fid, const char *name, int mode,
kgid_t gid, struct p9_qid *);
int p9_client_lock_dotl(struct p9_fid *fid, struct p9_flock *flock, u8 *status);
int p9_client_getlock_dotl(struct p9_fid *fid, struct p9_getlock *fl);
#define __HCI_CORE_H
#include <linux/leds.h>
+#include <linux/rculist.h>
+
#include <net/bluetooth/hci.h>
#include <net/bluetooth/hci_sock.h>
#define _LINUX_NET_BUSY_POLL_H
#include <linux/netdevice.h>
+#include <linux/sched/clock.h>
+#include <linux/sched/signal.h>
#include <net/ip.h>
#ifdef CONFIG_NET_RX_BUSY_POLL
* Slot timer must never exceed 85 ms, and must always be at least 25 ms,
* suggested to 75-85 msec by IrDA lite. This doesn't work with a lot of
* devices, and other stackes uses a lot more, so it's best we do it as well
- * (Note : this is the default value and sysctl overides it - Jean II)
+ * (Note : this is the default value and sysctl overrides it - Jean II)
*/
#define SLOT_TIMEOUT (90*HZ/1000)
const struct nlattr *nla, u32 objtype,
u8 genmask);
-int nft_obj_notify(struct net *net, struct nft_table *table,
- struct nft_object *obj, u32 portid, u32 seq,
- int event, int family, int report, gfp_t gfp);
+void nft_obj_notify(struct net *net, struct nft_table *table,
+ struct nft_object *obj, u32 portid, u32 seq,
+ int event, int family, int report, gfp_t gfp);
/**
* struct nft_object_type - stateful object type
#include <linux/limits.h>
#include <linux/net.h>
+#include <linux/cred.h>
#include <linux/security.h>
#include <linux/pid.h>
#include <linux/nsproxy.h>
void sk_free(struct sock *sk);
void sk_destruct(struct sock *sk);
struct sock *sk_clone_lock(const struct sock *sk, const gfp_t priority);
+void sk_free_unlock_clone(struct sock *sk);
struct sk_buff *sock_wmalloc(struct sock *sk, unsigned long size, int force,
gfp_t priority);
#include <linux/types.h>
#include <linux/sched.h>
+#include <linux/cred.h>
struct ib_addr {
union {
extern int scsi_unregister_device_handler(struct scsi_device_handler *scsi_dh);
void scsi_attach_vpd(struct scsi_device *sdev);
+extern struct scsi_device *scsi_device_from_queue(struct request_queue *q);
extern int scsi_device_get(struct scsi_device *);
extern void scsi_device_put(struct scsi_device *);
extern struct scsi_device *scsi_device_lookup(struct Scsi_Host *,
extern void scsi_sanitize_inquiry_string(unsigned char *s, int len);
extern int scsi_execute(struct scsi_device *sdev, const unsigned char *cmd,
int data_direction, void *buffer, unsigned bufflen,
- unsigned char *sense, int timeout, int retries,
- u64 flags, int *resid);
-extern int scsi_execute_req_flags(struct scsi_device *sdev,
- const unsigned char *cmd, int data_direction, void *buffer,
- unsigned bufflen, struct scsi_sense_hdr *sshdr, int timeout,
- int retries, int *resid, u64 flags, req_flags_t rq_flags);
+ unsigned char *sense, struct scsi_sense_hdr *sshdr,
+ int timeout, int retries, u64 flags,
+ req_flags_t rq_flags, int *resid);
static inline int scsi_execute_req(struct scsi_device *sdev,
const unsigned char *cmd, int data_direction, void *buffer,
unsigned bufflen, struct scsi_sense_hdr *sshdr, int timeout,
int retries, int *resid)
{
- return scsi_execute_req_flags(sdev, cmd, data_direction, buffer,
- bufflen, sshdr, timeout, retries, resid, 0, 0);
+ return scsi_execute(sdev, cmd, data_direction, buffer,
+ bufflen, NULL, sshdr, timeout, retries, 0, 0, resid);
}
extern void sdev_disable_disk_events(struct scsi_device *sdev);
extern void sdev_enable_disk_events(struct scsi_device *sdev);
*
*/
+#include <linux/wait.h>
#include <sound/asound.h>
#define snd_kcontrol_chip(kcontrol) ((kcontrol)->private_data)
extern void iscsit_set_unsoliticed_dataout(struct iscsi_cmd *);
extern int iscsit_process_scsi_cmd(struct iscsi_conn *, struct iscsi_cmd *,
struct iscsi_scsi_req *);
-extern int iscsit_check_dataout_hdr(struct iscsi_conn *, unsigned char *,
- struct iscsi_cmd **);
+extern int
+__iscsit_check_dataout_hdr(struct iscsi_conn *, void *,
+ struct iscsi_cmd *, u32, bool *);
+extern int
+iscsit_check_dataout_hdr(struct iscsi_conn *conn, void *buf,
+ struct iscsi_cmd **out_cmd);
extern int iscsit_check_dataout_payload(struct iscsi_cmd *, struct iscsi_data *,
bool);
extern int iscsit_setup_nop_out(struct iscsi_conn *, struct iscsi_cmd *,
extern void iscsit_free_cmd(struct iscsi_cmd *, bool);
extern void iscsit_add_cmd_to_immediate_queue(struct iscsi_cmd *,
struct iscsi_conn *, u8);
+extern struct iscsi_cmd *
+iscsit_find_cmd_from_itt_or_dump(struct iscsi_conn *conn,
+ itt_t init_task_tag, u32 length);
/*
* From iscsi_target_nego.c
#include <linux/configfs.h> /* struct config_group */
#include <linux/dma-direction.h> /* enum dma_data_direction */
#include <linux/percpu_ida.h> /* struct percpu_ida */
+#include <linux/percpu-refcount.h>
#include <linux/semaphore.h> /* struct semaphore */
+#include <linux/completion.h>
#define TARGET_CORE_VERSION "v5.0"
TMR_LUN_RESET = 5,
TMR_TARGET_WARM_RESET = 6,
TMR_TARGET_COLD_RESET = 7,
+ TMR_UNKNOWN = 0xff,
};
/* fabric independent task management response values */
void *fabric_tmr_ptr;
struct se_cmd *task_cmd;
struct se_device *tmr_dev;
- struct se_lun *tmr_lun;
struct list_head tmr_list;
};
#define CMD_T_COMPLETE (1 << 2)
#define CMD_T_SENT (1 << 4)
#define CMD_T_STOP (1 << 5)
-#define CMD_T_DEV_ACTIVE (1 << 7)
-#define CMD_T_BUSY (1 << 9)
#define CMD_T_TAS (1 << 10)
#define CMD_T_FABRIC_STOP (1 << 11)
spinlock_t t_state_lock;
struct config_group lun_group;
struct se_port_stat_grps port_stat_grps;
struct completion lun_ref_comp;
+ struct completion lun_shutdown_comp;
struct percpu_ref lun_ref;
struct list_head lun_dev_link;
struct hlist_node link;
u32 dev_index;
u64 creation_time;
atomic_long_t num_resets;
+ atomic_long_t aborts_complete;
+ atomic_long_t aborts_no_task;
atomic_long_t num_cmds;
atomic_long_t read_bytes;
atomic_long_t write_bytes;
u32 (*tpg_get_inst_index)(struct se_portal_group *);
/*
* Optional to release struct se_cmd and fabric dependent allocated
- * I/O descriptor in transport_cmd_check_stop().
+ * I/O descriptor after command execution has finished.
*
* Returning 1 will signal a descriptor has been released.
* Returning 0 will signal a descriptor has not been released.
rxrpc_recvmsg_full,
rxrpc_recvmsg_hole,
rxrpc_recvmsg_next,
+ rxrpc_recvmsg_requeue,
rxrpc_recvmsg_return,
rxrpc_recvmsg_terminal,
rxrpc_recvmsg_to_be_accepted,
EM(rxrpc_recvmsg_full, "FULL") \
EM(rxrpc_recvmsg_hole, "HOLE") \
EM(rxrpc_recvmsg_next, "NEXT") \
+ EM(rxrpc_recvmsg_requeue, "REQU") \
EM(rxrpc_recvmsg_return, "RETN") \
EM(rxrpc_recvmsg_terminal, "TERM") \
EM(rxrpc_recvmsg_to_be_accepted, "TBAC") \
#if !defined(_TRACE_SCHED_H) || defined(TRACE_HEADER_MULTI_READ)
#define _TRACE_SCHED_H
-#include <linux/sched.h>
+#include <linux/sched/numa_balancing.h>
#include <linux/tracepoint.h>
#include <linux/binfmts.h>
#undef TRACE_SYSTEM
#define TRACE_SYSTEM raw_syscalls
+#undef TRACE_INCLUDE_FILE
#define TRACE_INCLUDE_FILE syscalls
#if !defined(_TRACE_EVENTS_SYSCALLS_H) || defined(TRACE_HEADER_MULTI_READ)
header-y += virtio_gpu.h
header-y += virtio_ids.h
header-y += virtio_input.h
+header-y += virtio_mmio.h
header-y += virtio_net.h
header-y += virtio_pci.h
header-y += virtio_ring.h
#define AT_NO_AUTOMOUNT 0x800 /* Suppress terminal automount traversal */
#define AT_EMPTY_PATH 0x1000 /* Allow empty relative pathname */
+#define AT_STATX_SYNC_TYPE 0x6000 /* Type of synchronisation required from statx() */
+#define AT_STATX_SYNC_AS_STAT 0x0000 /* - Do whatever stat() does */
+#define AT_STATX_FORCE_SYNC 0x2000 /* - Force the attributes to be sync'd with the server */
+#define AT_STATX_DONT_SYNC 0x4000 /* - Don't sync attributes with the server */
+
#endif /* _UAPI_LINUX_FCNTL_H */
--- /dev/null
+#ifndef _UAPI_LINUX_SCHED_TYPES_H
+#define _UAPI_LINUX_SCHED_TYPES_H
+
+#include <linux/types.h>
+
+struct sched_param {
+ int sched_priority;
+};
+
+#define SCHED_ATTR_SIZE_VER0 48 /* sizeof first published struct */
+
+/*
+ * Extended scheduling parameters data structure.
+ *
+ * This is needed because the original struct sched_param can not be
+ * altered without introducing ABI issues with legacy applications
+ * (e.g., in sched_getparam()).
+ *
+ * However, the possibility of specifying more than just a priority for
+ * the tasks may be useful for a wide variety of application fields, e.g.,
+ * multimedia, streaming, automation and control, and many others.
+ *
+ * This variant (sched_attr) is meant at describing a so-called
+ * sporadic time-constrained task. In such model a task is specified by:
+ * - the activation period or minimum instance inter-arrival time;
+ * - the maximum (or average, depending on the actual scheduling
+ * discipline) computation time of all instances, a.k.a. runtime;
+ * - the deadline (relative to the actual activation time) of each
+ * instance.
+ * Very briefly, a periodic (sporadic) task asks for the execution of
+ * some specific computation --which is typically called an instance--
+ * (at most) every period. Moreover, each instance typically lasts no more
+ * than the runtime and must be completed by time instant t equal to
+ * the instance activation time + the deadline.
+ *
+ * This is reflected by the actual fields of the sched_attr structure:
+ *
+ * @size size of the structure, for fwd/bwd compat.
+ *
+ * @sched_policy task's scheduling policy
+ * @sched_flags for customizing the scheduler behaviour
+ * @sched_nice task's nice value (SCHED_NORMAL/BATCH)
+ * @sched_priority task's static priority (SCHED_FIFO/RR)
+ * @sched_deadline representative of the task's deadline
+ * @sched_runtime representative of the task's runtime
+ * @sched_period representative of the task's period
+ *
+ * Given this task model, there are a multiplicity of scheduling algorithms
+ * and policies, that can be used to ensure all the tasks will make their
+ * timing constraints.
+ *
+ * As of now, the SCHED_DEADLINE policy (sched_dl scheduling class) is the
+ * only user of this new interface. More information about the algorithm
+ * available in the scheduling class file or in Documentation/.
+ */
+struct sched_attr {
+ u32 size;
+
+ u32 sched_policy;
+ u64 sched_flags;
+
+ /* SCHED_NORMAL, SCHED_BATCH */
+ s32 sched_nice;
+
+ /* SCHED_FIFO, SCHED_RR */
+ u32 sched_priority;
+
+ /* SCHED_DEADLINE */
+ u64 sched_runtime;
+ u64 sched_deadline;
+ u64 sched_period;
+};
+
+#endif /* _UAPI_LINUX_SCHED_TYPES_H */
#ifndef _UAPI_LINUX_STAT_H
#define _UAPI_LINUX_STAT_H
+#include <linux/types.h>
#if defined(__KERNEL__) || !defined(__GLIBC__) || (__GLIBC__ < 2)
#endif
+/*
+ * Timestamp structure for the timestamps in struct statx.
+ *
+ * tv_sec holds the number of seconds before (negative) or after (positive)
+ * 00:00:00 1st January 1970 UTC.
+ *
+ * tv_nsec holds a number of nanoseconds before (0..-999,999,999 if tv_sec is
+ * negative) or after (0..999,999,999 if tv_sec is positive) the tv_sec time.
+ *
+ * Note that if both tv_sec and tv_nsec are non-zero, then the two values must
+ * either be both positive or both negative.
+ *
+ * __reserved is held in case we need a yet finer resolution.
+ */
+struct statx_timestamp {
+ __s64 tv_sec;
+ __s32 tv_nsec;
+ __s32 __reserved;
+};
+
+/*
+ * Structures for the extended file attribute retrieval system call
+ * (statx()).
+ *
+ * The caller passes a mask of what they're specifically interested in as a
+ * parameter to statx(). What statx() actually got will be indicated in
+ * st_mask upon return.
+ *
+ * For each bit in the mask argument:
+ *
+ * - if the datum is not supported:
+ *
+ * - the bit will be cleared, and
+ *
+ * - the datum will be set to an appropriate fabricated value if one is
+ * available (eg. CIFS can take a default uid and gid), otherwise
+ *
+ * - the field will be cleared;
+ *
+ * - otherwise, if explicitly requested:
+ *
+ * - the datum will be synchronised to the server if AT_STATX_FORCE_SYNC is
+ * set or if the datum is considered out of date, and
+ *
+ * - the field will be filled in and the bit will be set;
+ *
+ * - otherwise, if not requested, but available in approximate form without any
+ * effort, it will be filled in anyway, and the bit will be set upon return
+ * (it might not be up to date, however, and no attempt will be made to
+ * synchronise the internal state first);
+ *
+ * - otherwise the field and the bit will be cleared before returning.
+ *
+ * Items in STATX_BASIC_STATS may be marked unavailable on return, but they
+ * will have values installed for compatibility purposes so that stat() and
+ * co. can be emulated in userspace.
+ */
+struct statx {
+ /* 0x00 */
+ __u32 stx_mask; /* What results were written [uncond] */
+ __u32 stx_blksize; /* Preferred general I/O size [uncond] */
+ __u64 stx_attributes; /* Flags conveying information about the file [uncond] */
+ /* 0x10 */
+ __u32 stx_nlink; /* Number of hard links */
+ __u32 stx_uid; /* User ID of owner */
+ __u32 stx_gid; /* Group ID of owner */
+ __u16 stx_mode; /* File mode */
+ __u16 __spare0[1];
+ /* 0x20 */
+ __u64 stx_ino; /* Inode number */
+ __u64 stx_size; /* File size */
+ __u64 stx_blocks; /* Number of 512-byte blocks allocated */
+ __u64 __spare1[1];
+ /* 0x40 */
+ struct statx_timestamp stx_atime; /* Last access time */
+ struct statx_timestamp stx_btime; /* File creation time */
+ struct statx_timestamp stx_ctime; /* Last attribute change time */
+ struct statx_timestamp stx_mtime; /* Last data modification time */
+ /* 0x80 */
+ __u32 stx_rdev_major; /* Device ID of special file [if bdev/cdev] */
+ __u32 stx_rdev_minor;
+ __u32 stx_dev_major; /* ID of device containing file [uncond] */
+ __u32 stx_dev_minor;
+ /* 0x90 */
+ __u64 __spare2[14]; /* Spare space for future expansion */
+ /* 0x100 */
+};
+
+/*
+ * Flags to be stx_mask
+ *
+ * Query request/result mask for statx() and struct statx::stx_mask.
+ *
+ * These bits should be set in the mask argument of statx() to request
+ * particular items when calling statx().
+ */
+#define STATX_TYPE 0x00000001U /* Want/got stx_mode & S_IFMT */
+#define STATX_MODE 0x00000002U /* Want/got stx_mode & ~S_IFMT */
+#define STATX_NLINK 0x00000004U /* Want/got stx_nlink */
+#define STATX_UID 0x00000008U /* Want/got stx_uid */
+#define STATX_GID 0x00000010U /* Want/got stx_gid */
+#define STATX_ATIME 0x00000020U /* Want/got stx_atime */
+#define STATX_MTIME 0x00000040U /* Want/got stx_mtime */
+#define STATX_CTIME 0x00000080U /* Want/got stx_ctime */
+#define STATX_INO 0x00000100U /* Want/got stx_ino */
+#define STATX_SIZE 0x00000200U /* Want/got stx_size */
+#define STATX_BLOCKS 0x00000400U /* Want/got stx_blocks */
+#define STATX_BASIC_STATS 0x000007ffU /* The stuff in the normal stat struct */
+#define STATX_BTIME 0x00000800U /* Want/got stx_btime */
+#define STATX_ALL 0x00000fffU /* All currently supported flags */
+
+/*
+ * Attributes to be found in stx_attributes
+ *
+ * These give information about the features or the state of a file that might
+ * be of use to ordinary userspace programs such as GUIs or ls rather than
+ * specialised tools.
+ *
+ * Note that the flags marked [I] correspond to generic FS_IOC_FLAGS
+ * semantically. Where possible, the numerical value is picked to correspond
+ * also.
+ */
+#define STATX_ATTR_COMPRESSED 0x00000004 /* [I] File is compressed by the fs */
+#define STATX_ATTR_IMMUTABLE 0x00000010 /* [I] File is marked immutable */
+#define STATX_ATTR_APPEND 0x00000020 /* [I] File is append-only */
+#define STATX_ATTR_NODUMP 0x00000040 /* [I] File is not to be dumped */
+#define STATX_ATTR_ENCRYPTED 0x00000800 /* [I] File requires key to decrypt in fs */
+
+#define STATX_ATTR_AUTOMOUNT 0x00001000 /* Dir: Automount trigger */
+
#endif /* _UAPI_LINUX_STAT_H */
union {
struct {
- uint32_t iov_cnt;
- uint32_t iov_bidi_cnt;
- uint32_t iov_dif_cnt;
- uint64_t cdb_off;
- uint64_t __pad1;
- uint64_t __pad2;
+ __u32 iov_cnt;
+ __u32 iov_bidi_cnt;
+ __u32 iov_dif_cnt;
+ __u64 cdb_off;
+ __u64 __pad1;
+ __u64 __pad2;
struct iovec iov[0];
} req;
struct {
- uint8_t scsi_status;
- uint8_t __pad1;
- uint16_t __pad2;
- uint32_t __pad3;
+ __u8 scsi_status;
+ __u8 __pad1;
+ __u16 __pad2;
+ __u32 __pad3;
char sense_buffer[TCMU_SENSE_BUFFERSIZE];
} rsp;
};
} __packed;
-#define TCMU_OP_ALIGN_SIZE sizeof(uint64_t)
+#define TCMU_OP_ALIGN_SIZE sizeof(__u64)
enum tcmu_genl_cmd {
TCMU_CMD_UNSPEC,
* means the userland is reading).
*/
#define UFFD_API ((__u64)0xAA)
-#define UFFD_API_FEATURES (UFFD_FEATURE_EVENT_EXIT | \
- UFFD_FEATURE_EVENT_FORK | \
+#define UFFD_API_FEATURES (UFFD_FEATURE_EVENT_FORK | \
UFFD_FEATURE_EVENT_REMAP | \
UFFD_FEATURE_EVENT_REMOVE | \
UFFD_FEATURE_EVENT_UNMAP | \
#define UFFD_EVENT_REMAP 0x14
#define UFFD_EVENT_REMOVE 0x15
#define UFFD_EVENT_UNMAP 0x16
-#define UFFD_EVENT_EXIT 0x17
/* flags for UFFD_EVENT_PAGEFAULT */
#define UFFD_PAGEFAULT_FLAG_WRITE (1<<0) /* If this was a write fault */
#define UFFD_FEATURE_MISSING_HUGETLBFS (1<<4)
#define UFFD_FEATURE_MISSING_SHMEM (1<<5)
#define UFFD_FEATURE_EVENT_UNMAP (1<<6)
-#define UFFD_FEATURE_EVENT_EXIT (1<<7)
__u64 features;
__u64 ioctls;
--- /dev/null
+/*
+ * Virtio platform device driver
+ *
+ * Copyright 2011, ARM Ltd.
+ *
+ * Based on Virtio PCI driver by Anthony Liguori, copyright IBM Corp. 2007
+ *
+ * This header is BSD licensed so anyone can use the definitions to implement
+ * compatible drivers/servers.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ * 3. Neither the name of IBM nor the names of its contributors
+ * may be used to endorse or promote products derived from this software
+ * without specific prior written permission.
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL IBM OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#ifndef _LINUX_VIRTIO_MMIO_H
+#define _LINUX_VIRTIO_MMIO_H
+
+/*
+ * Control registers
+ */
+
+/* Magic value ("virt" string) - Read Only */
+#define VIRTIO_MMIO_MAGIC_VALUE 0x000
+
+/* Virtio device version - Read Only */
+#define VIRTIO_MMIO_VERSION 0x004
+
+/* Virtio device ID - Read Only */
+#define VIRTIO_MMIO_DEVICE_ID 0x008
+
+/* Virtio vendor ID - Read Only */
+#define VIRTIO_MMIO_VENDOR_ID 0x00c
+
+/* Bitmask of the features supported by the device (host)
+ * (32 bits per set) - Read Only */
+#define VIRTIO_MMIO_DEVICE_FEATURES 0x010
+
+/* Device (host) features set selector - Write Only */
+#define VIRTIO_MMIO_DEVICE_FEATURES_SEL 0x014
+
+/* Bitmask of features activated by the driver (guest)
+ * (32 bits per set) - Write Only */
+#define VIRTIO_MMIO_DRIVER_FEATURES 0x020
+
+/* Activated features set selector - Write Only */
+#define VIRTIO_MMIO_DRIVER_FEATURES_SEL 0x024
+
+
+#ifndef VIRTIO_MMIO_NO_LEGACY /* LEGACY DEVICES ONLY! */
+
+/* Guest's memory page size in bytes - Write Only */
+#define VIRTIO_MMIO_GUEST_PAGE_SIZE 0x028
+
+#endif
+
+
+/* Queue selector - Write Only */
+#define VIRTIO_MMIO_QUEUE_SEL 0x030
+
+/* Maximum size of the currently selected queue - Read Only */
+#define VIRTIO_MMIO_QUEUE_NUM_MAX 0x034
+
+/* Queue size for the currently selected queue - Write Only */
+#define VIRTIO_MMIO_QUEUE_NUM 0x038
+
+
+#ifndef VIRTIO_MMIO_NO_LEGACY /* LEGACY DEVICES ONLY! */
+
+/* Used Ring alignment for the currently selected queue - Write Only */
+#define VIRTIO_MMIO_QUEUE_ALIGN 0x03c
+
+/* Guest's PFN for the currently selected queue - Read Write */
+#define VIRTIO_MMIO_QUEUE_PFN 0x040
+
+#endif
+
+
+/* Ready bit for the currently selected queue - Read Write */
+#define VIRTIO_MMIO_QUEUE_READY 0x044
+
+/* Queue notifier - Write Only */
+#define VIRTIO_MMIO_QUEUE_NOTIFY 0x050
+
+/* Interrupt status - Read Only */
+#define VIRTIO_MMIO_INTERRUPT_STATUS 0x060
+
+/* Interrupt acknowledge - Write Only */
+#define VIRTIO_MMIO_INTERRUPT_ACK 0x064
+
+/* Device status register - Read Write */
+#define VIRTIO_MMIO_STATUS 0x070
+
+/* Selected queue's Descriptor Table address, 64 bits in two halves */
+#define VIRTIO_MMIO_QUEUE_DESC_LOW 0x080
+#define VIRTIO_MMIO_QUEUE_DESC_HIGH 0x084
+
+/* Selected queue's Available Ring address, 64 bits in two halves */
+#define VIRTIO_MMIO_QUEUE_AVAIL_LOW 0x090
+#define VIRTIO_MMIO_QUEUE_AVAIL_HIGH 0x094
+
+/* Selected queue's Used Ring address, 64 bits in two halves */
+#define VIRTIO_MMIO_QUEUE_USED_LOW 0x0a0
+#define VIRTIO_MMIO_QUEUE_USED_HIGH 0x0a4
+
+/* Configuration atomicity value */
+#define VIRTIO_MMIO_CONFIG_GENERATION 0x0fc
+
+/* The config space is defined by each driver as
+ * the per-driver configuration space - Read Write */
+#define VIRTIO_MMIO_CONFIG 0x100
+
+
+
+/*
+ * Interrupt flags (re: interrupt status & acknowledge registers)
+ */
+
+#define VIRTIO_MMIO_INT_VRING (1 << 0)
+#define VIRTIO_MMIO_INT_CONFIG (1 << 1)
+
+#endif
* configuration space */
#define VIRTIO_PCI_CONFIG_OFF(msix_enabled) ((msix_enabled) ? 24 : 20)
/* Deprecated: please use VIRTIO_PCI_CONFIG_OFF instead */
-#define VIRTIO_PCI_CONFIG(dev) VIRTIO_PCI_CONFIG_OFF((dev)->msix_enabled)
+#define VIRTIO_PCI_CONFIG(dev) VIRTIO_PCI_CONFIG_OFF((dev)->pci_dev->msix_enabled)
/* Virtio ABI version, this must match exactly */
#define VIRTIO_PCI_ABI_VERSION 0
#define __LINUX_SWIOTLB_XEN_H
#include <linux/dma-direction.h>
+#include <linux/scatterlist.h>
#include <linux/swiotlb.h>
extern int xen_swiotlb_init(int verbose, bool early);
extern int
xen_swiotlb_set_dma_mask(struct device *dev, u64 dma_mask);
+
+extern int
+xen_swiotlb_dma_mmap(struct device *dev, struct vm_area_struct *vma,
+ void *cpu_addr, dma_addr_t dma_addr, size_t size,
+ unsigned long attrs);
+
+extern int
+xen_swiotlb_get_sgtable(struct device *dev, struct sg_table *sgt,
+ void *cpu_addr, dma_addr_t handle, size_t size,
+ unsigned long attrs);
#endif /* __LINUX_SWIOTLB_XEN_H */
#include <linux/sched.h>
#include <linux/sched/sysctl.h>
#include <linux/sched/rt.h>
+#include <linux/sched/task.h>
#include <linux/init.h>
#include <linux/fs.h>
#include <linux/mm.h>
#include <linux/extable.h>
#include <linux/module.h>
#include <linux/proc_fs.h>
+#include <linux/binfmts.h>
#include <linux/kernel.h>
#include <linux/syscalls.h>
#include <linux/stackprotector.h>
#include <linux/bootmem.h>
#include <linux/acpi.h>
#include <linux/tty.h>
+#include <linux/nmi.h>
#include <linux/percpu.h>
#include <linux/kmod.h>
#include <linux/vmalloc.h>
#include <linux/device.h>
#include <linux/kthread.h>
#include <linux/sched.h>
+#include <linux/sched/init.h>
#include <linux/signal.h>
#include <linux/idr.h>
#include <linux/kgdb.h>
#include <linux/blkdev.h>
#include <linux/elevator.h>
#include <linux/sched_clock.h>
+#include <linux/sched/task.h>
+#include <linux/sched/task_stack.h>
#include <linux/context_tracking.h>
#include <linux/random.h>
#include <linux/list.h>
do_ctors();
usermodehelper_enable();
do_initcalls();
- random_int_secret_init();
}
static void __init do_pre_smp_initcalls(void)
#include <linux/ipc_namespace.h>
#include <linux/user_namespace.h>
#include <linux/slab.h>
+#include <linux/sched/wake_q.h>
+#include <linux/sched/signal.h>
+#include <linux/sched/user.h>
#include <net/sock.h>
#include "util.h"
#include <linux/proc_fs.h>
#include <linux/list.h>
#include <linux/security.h>
-#include <linux/sched.h>
+#include <linux/sched/wake_q.h>
#include <linux/syscalls.h>
#include <linux/audit.h>
#include <linux/seq_file.h>
#include <linux/rcupdate.h>
#include <linux/nsproxy.h>
#include <linux/slab.h>
+#include <linux/cred.h>
#include <linux/fs.h>
#include <linux/mount.h>
#include <linux/user_namespace.h>
#include <linux/proc_ns.h>
+#include <linux/sched/task.h>
#include "util.h"
#include <linux/rwsem.h>
#include <linux/nsproxy.h>
#include <linux/ipc_namespace.h>
+#include <linux/sched/wake_q.h>
#include <linux/uaccess.h>
#include "util.h"
if (ret)
return ret;
- ret = sfd->file->f_op->mmap(sfd->file, vma);
+ ret = call_mmap(sfd->file, vma);
if (ret) {
shm_close(vma);
return ret;
if (!sfd->file->f_op->fsync)
return -EINVAL;
- return sfd->file->f_op->fsync(sfd->file, start, end, datasync);
+ return call_fsync(sfd->file, start, end, datasync);
}
static long shm_fallocate(struct file *file, int mode, loff_t offset,
#include <linux/syscalls.h>
#include <linux/mount.h>
#include <linux/uaccess.h>
+#include <linux/sched/cputime.h>
+
#include <asm/div64.h>
#include <linux/blkdev.h> /* sector_div */
#include <linux/pid_namespace.h>
#include <linux/bpf_trace.h>
#include <linux/syscalls.h>
#include <linux/slab.h>
+#include <linux/sched/signal.h>
#include <linux/vmalloc.h>
#include <linux/mmzone.h>
#include <linux/anon_inodes.h>
* - out of bounds or malformed jumps
* The second pass is all possible path descent from the 1st insn.
* Since it's analyzing all pathes through the program, the length of the
- * analysis is limited to 32k insn, which may be hit even if total number of
+ * analysis is limited to 64k insn, which may be hit even if total number of
* insn is less then 4K, but there are too many branches that change stack/regs.
* Number of 'branches to be analyzed' is limited to 1k
*
#include <linux/sort.h>
#include <linux/delay.h>
#include <linux/mm.h>
+#include <linux/sched/signal.h>
+#include <linux/sched/task.h>
+#include <linux/magic.h>
#include <linux/slab.h>
#include <linux/vmalloc.h>
#include <linux/delayacct.h>
#include <linux/proc_fs.h>
#include <linux/rcupdate.h>
#include <linux/sched.h>
+#include <linux/sched/task.h>
#include <linux/slab.h>
#include <linux/spinlock.h>
#include <linux/percpu-rwsem.h>
*
* Returns 0 on success, -errno on failure. On failure, csses which have
* been processed already aren't cleaned up. The caller is responsible for
- * cleaning up with cgroup_apply_control_disble().
+ * cleaning up with cgroup_apply_control_disable().
*/
static int cgroup_apply_control_enable(struct cgroup *cgrp)
{
#include <linux/proc_fs.h>
#include <linux/rcupdate.h>
#include <linux/sched.h>
+#include <linux/sched/mm.h>
+#include <linux/sched/task.h>
#include <linux/seq_file.h>
#include <linux/security.h>
#include <linux/slab.h>
#include "cgroup-internal.h"
-#include <linux/sched.h>
+#include <linux/sched/task.h>
#include <linux/slab.h>
#include <linux/nsproxy.h>
#include <linux/proc_ns.h>
/*
* task_css_check(true) in pids_can_fork() and pids_cancel_fork() relies
- * on threadgroup_change_begin() held by the copy_process().
+ * on cgroup_threadgroup_change_begin() held by the copy_process().
*/
static int pids_can_fork(struct task_struct *task)
{
#include <linux/smp.h>
#include <linux/init.h>
#include <linux/notifier.h>
-#include <linux/sched.h>
+#include <linux/sched/signal.h>
+#include <linux/sched/hotplug.h>
+#include <linux/sched/task.h>
#include <linux/unistd.h>
#include <linux/cpu.h>
#include <linux/oom.h>
#include <linux/cred.h>
#include <linux/slab.h>
#include <linux/sched.h>
+#include <linux/sched/coredump.h>
#include <linux/key.h>
#include <linux/keyctl.h>
#include <linux/init_task.h>
#include <linux/init.h>
#include <linux/kgdb.h>
#include <linux/kdb.h>
+#include <linux/nmi.h>
#include <linux/pid.h>
#include <linux/smp.h>
#include <linux/mm.h>
int i;
for (i = 0; i < VMACACHE_SIZE; i++) {
- if (!current->vmacache[i])
+ if (!current->vmacache.vmas[i])
continue;
- flush_cache_range(current->vmacache[i],
+ flush_cache_range(current->vmacache.vmas[i],
addr, addr + BREAK_INSTR_SIZE);
}
}
*/
#include <linux/kernel.h>
+#include <linux/sched/signal.h>
#include <linux/kgdb.h>
#include <linux/kdb.h>
#include <linux/serial_core.h>
#include <linux/ctype.h>
#include <linux/string.h>
#include <linux/kernel.h>
-#include <linux/sched.h>
+#include <linux/sched/signal.h>
+#include <linux/sched/debug.h>
#include <linux/kdb.h>
#include <linux/nmi.h>
#include "kdb_private.h"
#include <linux/kmsg_dump.h>
#include <linux/reboot.h>
#include <linux/sched.h>
+#include <linux/sched/loadavg.h>
+#include <linux/sched/stat.h>
+#include <linux/sched/debug.h>
#include <linux/sysrq.h>
#include <linux/smp.h>
#include <linux/utsname.h>
*/
#include <linux/sched.h>
+#include <linux/sched/task.h>
+#include <linux/sched/cputime.h>
#include <linux/slab.h>
#include <linux/taskstats.h>
#include <linux/time.h>
#include <linux/perf_event.h>
#include <linux/slab.h>
+#include <linux/sched/task_stack.h>
+
#include "internal.h"
struct callchain_cpus_entries {
#include <linux/filter.h>
#include <linux/namei.h>
#include <linux/parser.h>
+#include <linux/sched/clock.h>
+#include <linux/sched/mm.h>
#include "internal.h"
*/
#define PERF_CPU_HRTIMER (1000 / HZ)
/*
- * function must be called with interrupts disbled
+ * function must be called with interrupts disabled
*/
static enum hrtimer_restart perf_mux_hrtimer_handler(struct hrtimer *hr)
{
#include <linux/pagemap.h> /* read_mapping_page */
#include <linux/slab.h>
#include <linux/sched.h>
+#include <linux/sched/mm.h>
+#include <linux/sched/coredump.h>
#include <linux/export.h>
#include <linux/rmap.h> /* anon_vma_prepare */
#include <linux/mmu_notifier.h> /* set_pte_at_notify */
#include <linux/mm.h>
#include <linux/slab.h>
+#include <linux/sched/autogroup.h>
+#include <linux/sched/mm.h>
+#include <linux/sched/stat.h>
+#include <linux/sched/task.h>
+#include <linux/sched/task_stack.h>
+#include <linux/sched/cputime.h>
#include <linux/interrupt.h>
#include <linux/module.h>
#include <linux/capability.h>
enter_lazy_tlb(mm, current);
task_unlock(current);
mm_update_next_owner(mm);
- userfaultfd_exit(mm);
mmput(mm);
if (test_thread_flag(TIF_MEMDIE))
exit_oom_victim();
*/
#include <linux/slab.h>
+#include <linux/sched/autogroup.h>
+#include <linux/sched/mm.h>
+#include <linux/sched/coredump.h>
+#include <linux/sched/user.h>
+#include <linux/sched/numa_balancing.h>
+#include <linux/sched/stat.h>
+#include <linux/sched/task.h>
+#include <linux/sched/task_stack.h>
+#include <linux/sched/cputime.h>
+#include <linux/rtmutex.h>
#include <linux/init.h>
#include <linux/unistd.h>
#include <linux/module.h>
task->pids[type].pid = pid;
}
+static inline void rcu_copy_process(struct task_struct *p)
+{
+#ifdef CONFIG_PREEMPT_RCU
+ p->rcu_read_lock_nesting = 0;
+ p->rcu_read_unlock_special.s = 0;
+ p->rcu_blocked_node = NULL;
+ INIT_LIST_HEAD(&p->rcu_node_entry);
+#endif /* #ifdef CONFIG_PREEMPT_RCU */
+#ifdef CONFIG_TASKS_RCU
+ p->rcu_tasks_holdout = false;
+ INIT_LIST_HEAD(&p->rcu_tasks_holdout_list);
+ p->rcu_tasks_idle_cpu = -1;
+#endif /* #ifdef CONFIG_TASKS_RCU */
+}
+
/*
* This creates a new process as a copy of the old one,
* but does not actually start it yet.
INIT_LIST_HEAD(&p->thread_group);
p->task_works = NULL;
- threadgroup_change_begin(current);
+ cgroup_threadgroup_change_begin(current);
/*
* Ensure that the cgroup subsystem policies allow the new process to be
* forked. It should be noted the the new process's css_set can be changed
proc_fork_connector(p);
cgroup_post_fork(p);
- threadgroup_change_end(current);
+ cgroup_threadgroup_change_end(current);
perf_event_fork(p);
trace_task_newtask(p, clone_flags);
bad_fork_cancel_cgroup:
cgroup_cancel_fork(p);
bad_fork_free_pid:
- threadgroup_change_end(current);
+ cgroup_threadgroup_change_end(current);
if (pid != &init_struct_pid)
free_pid(pid);
bad_fork_cleanup_thread:
#include <linux/nsproxy.h>
#include <linux/ptrace.h>
#include <linux/sched/rt.h>
+#include <linux/sched/wake_q.h>
+#include <linux/sched/mm.h>
#include <linux/hugetlb.h>
#include <linux/freezer.h>
#include <linux/bootmem.h>
#include <linux/export.h>
#include <linux/sysctl.h>
#include <linux/utsname.h>
+#include <linux/sched/signal.h>
+#include <linux/sched/debug.h>
+
#include <trace/events/sched.h>
/*
#include <linux/slab.h>
#include <linux/sched.h>
#include <linux/sched/rt.h>
+#include <linux/sched/task.h>
+#include <uapi/linux/sched/types.h>
#include <linux/task_work.h>
#include "internals.h"
ret = crypto_shash_final(desc, digest);
if (ret)
goto out_free_digest;
- ret = kexec_purgatory_get_set_symbol(image, "sha_regions",
- sha_regions, sha_region_sz, 0);
+ ret = kexec_purgatory_get_set_symbol(image, "purgatory_sha_regions",
+ sha_regions, sha_region_sz, 0);
if (ret)
goto out_free_digest;
- ret = kexec_purgatory_get_set_symbol(image, "sha256_digest",
- digest, SHA256_DIGEST_SIZE, 0);
+ ret = kexec_purgatory_get_set_symbol(image, "purgatory_sha256_digest",
+ digest, SHA256_DIGEST_SIZE, 0);
if (ret)
goto out_free_digest;
}
extern struct mutex kexec_mutex;
#ifdef CONFIG_KEXEC_FILE
-struct kexec_sha_region {
- unsigned long start;
- unsigned long len;
-};
-
+#include <linux/purgatory.h>
void kimage_file_post_load_cleanup(struct kimage *image);
#else /* CONFIG_KEXEC_FILE */
static inline void kimage_file_post_load_cleanup(struct kimage *image) { }
*/
#include <linux/module.h>
#include <linux/sched.h>
+#include <linux/sched/task.h>
+#include <linux/binfmts.h>
#include <linux/syscalls.h>
#include <linux/unistd.h>
#include <linux/kmod.h>
* even if we're invoked from userspace (think modprobe, hotplug cpu,
* etc.).
*/
+#include <uapi/linux/sched/types.h>
#include <linux/sched.h>
+#include <linux/sched/task.h>
#include <linux/kthread.h>
#include <linux/completion.h>
#include <linux/err.h>
#include <linux/latencytop.h>
#include <linux/export.h>
#include <linux/sched.h>
+#include <linux/sched/debug.h>
+#include <linux/sched/stat.h>
#include <linux/list.h>
#include <linux/stacktrace.h>
#define DISABLE_BRANCH_PROFILING
#include <linux/mutex.h>
#include <linux/sched.h>
+#include <linux/sched/clock.h>
+#include <linux/sched/task.h>
#include <linux/delay.h>
#include <linux/module.h>
#include <linux/proc_fs.h>
if (depth) {
hlock = curr->held_locks + depth - 1;
if (hlock->class_idx == class_idx && nest_lock) {
- if (hlock->references)
+ if (hlock->references) {
+ /*
+ * Check: unsigned int references:12, overflow.
+ */
+ if (DEBUG_LOCKS_WARN_ON(hlock->references == (1 << 12)-1))
+ return 0;
+
hlock->references++;
- else
+ } else {
hlock->references = 2;
+ }
return 1;
}
#include <linux/smp.h>
#include <linux/interrupt.h>
#include <linux/sched.h>
+#include <uapi/linux/sched/types.h>
+#include <linux/rtmutex.h>
#include <linux/atomic.h>
#include <linux/moduleparam.h>
#include <linux/delay.h>
*/
#include <linux/mutex.h>
#include <linux/ww_mutex.h>
-#include <linux/sched.h>
+#include <linux/sched/signal.h>
#include <linux/sched/rt.h>
+#include <linux/sched/wake_q.h>
+#include <linux/sched/debug.h>
#include <linux/export.h>
#include <linux/spinlock.h>
#include <linux/interrupt.h>
*/
#include <linux/debugfs.h>
#include <linux/sched.h>
+#include <linux/sched/clock.h>
#include <linux/fs.h>
static const char * const qstat_names[qstat_num + 1] = {
*/
#include <linux/sched.h>
#include <linux/sched/rt.h>
+#include <linux/sched/debug.h>
#include <linux/delay.h>
#include <linux/export.h>
#include <linux/spinlock.h>
*/
#include <linux/spinlock.h>
#include <linux/export.h>
-#include <linux/sched.h>
+#include <linux/sched/signal.h>
#include <linux/sched/rt.h>
#include <linux/sched/deadline.h>
+#include <linux/sched/wake_q.h>
+#include <linux/sched/debug.h>
#include <linux/timer.h>
#include "rtmutex_common.h"
#define __KERNEL_RTMUTEX_COMMON_H
#include <linux/rtmutex.h>
+#include <linux/sched/wake_q.h>
/*
* This is the control structure for tasks blocked on a rt_mutex,
* - Derived also from comments by Linus
*/
#include <linux/rwsem.h>
-#include <linux/sched.h>
+#include <linux/sched/signal.h>
+#include <linux/sched/debug.h>
#include <linux/export.h>
enum rwsem_waiter_type {
* and Davidlohr Bueso <davidlohr@hp.com>. Based on mutexes.
*/
#include <linux/rwsem.h>
-#include <linux/sched.h>
#include <linux/init.h>
#include <linux/export.h>
+#include <linux/sched/signal.h>
#include <linux/sched/rt.h>
+#include <linux/sched/wake_q.h>
+#include <linux/sched/debug.h>
#include <linux/osq_lock.h>
#include "rwsem.h"
#include <linux/types.h>
#include <linux/kernel.h>
#include <linux/sched.h>
+#include <linux/sched/debug.h>
#include <linux/export.h>
#include <linux/rwsem.h>
#include <linux/atomic.h>
#include <linux/kernel.h>
#include <linux/export.h>
#include <linux/sched.h>
+#include <linux/sched/debug.h>
#include <linux/semaphore.h>
#include <linux/spinlock.h>
#include <linux/ftrace.h>
if (mtx->flags & TEST_MTX_TRY) {
while (!ww_mutex_trylock(&mtx->mutex))
- cpu_relax();
+ cond_resched();
} else {
ww_mutex_lock(&mtx->mutex, NULL);
}
ret = -EINVAL;
break;
}
- cpu_relax();
+ cond_resched();
} while (time_before(jiffies, timeout));
} else {
ret = wait_for_completion_timeout(&mtx.done, TIMEOUT);
if (ret)
return ret;
- ret = stress(4096, hweight32(STRESS_ALL)*ncpus, 1<<12, STRESS_ALL);
+ ret = stress(4095, hweight32(STRESS_ALL)*ncpus, 1<<12, STRESS_ALL);
if (ret)
return ret;
* to indicate a major problem.
*/
#include <linux/debug_locks.h>
+#include <linux/sched/debug.h>
#include <linux/interrupt.h>
#include <linux/kmsg_dump.h>
#include <linux/kallsyms.h>
#include <linux/syscalls.h>
#include <linux/proc_ns.h>
#include <linux/proc_fs.h>
+#include <linux/sched/task.h>
#define pid_hashfn(nr, ns) \
hash_long((unsigned long)nr + (unsigned long)ns, pidhash_shift)
#include <linux/pid_namespace.h>
#include <linux/user_namespace.h>
#include <linux/syscalls.h>
+#include <linux/cred.h>
#include <linux/err.h>
#include <linux/acct.h>
#include <linux/slab.h>
#include <linux/proc_ns.h>
#include <linux/reboot.h>
#include <linux/export.h>
+#include <linux/sched/task.h>
+#include <linux/sched/signal.h>
struct pid_cache {
int nr_ids;
* This file is released under the GPLv2.
*/
+#define pr_fmt(fmt) "PM: " fmt
+
#include <linux/export.h>
#include <linux/suspend.h>
#include <linux/syscalls.h>
#include <linux/fs.h>
#include <linux/mount.h>
#include <linux/pm.h>
+#include <linux/nmi.h>
#include <linux/console.h>
#include <linux/cpu.h>
#include <linux/freezer.h>
#ifdef CONFIG_PM_DEBUG
static void hibernation_debug_sleep(void)
{
- printk(KERN_INFO "hibernation debug: Waiting for 5 seconds.\n");
+ pr_info("hibernation debug: Waiting for 5 seconds.\n");
mdelay(5000);
}
centisecs = 1; /* avoid div-by-zero */
k = nr_pages * (PAGE_SIZE / 1024);
kps = (k * 100) / centisecs;
- printk(KERN_INFO "PM: %s %u kbytes in %u.%02u seconds (%u.%02u MB/s)\n",
- msg, k,
- centisecs / 100, centisecs % 100,
- kps / 1000, (kps % 1000) / 10);
+ pr_info("%s %u kbytes in %u.%02u seconds (%u.%02u MB/s)\n",
+ msg, k, centisecs / 100, centisecs % 100, kps / 1000,
+ (kps % 1000) / 10);
}
/**
error = dpm_suspend_end(PMSG_FREEZE);
if (error) {
- printk(KERN_ERR "PM: Some devices failed to power down, "
- "aborting hibernation\n");
+ pr_err("Some devices failed to power down, aborting hibernation\n");
return error;
}
error = syscore_suspend();
if (error) {
- printk(KERN_ERR "PM: Some system devices failed to power down, "
- "aborting hibernation\n");
+ pr_err("Some system devices failed to power down, aborting hibernation\n");
goto Enable_irqs;
}
restore_processor_state();
trace_suspend_resume(TPS("machine_suspend"), PM_EVENT_HIBERNATE, false);
if (error)
- printk(KERN_ERR "PM: Error %d creating hibernation image\n",
- error);
+ pr_err("Error %d creating hibernation image\n", error);
+
if (!in_suspend) {
events_check_enabled = false;
clear_free_pages();
error = dpm_suspend_end(PMSG_QUIESCE);
if (error) {
- printk(KERN_ERR "PM: Some devices failed to power down, "
- "aborting resume\n");
+ pr_err("Some devices failed to power down, aborting resume\n");
return error;
}
{
#ifdef CONFIG_SUSPEND
int error;
+
+ if (hibernation_mode == HIBERNATION_SUSPEND) {
+ error = suspend_devices_and_enter(PM_SUSPEND_MEM);
+ if (error) {
+ hibernation_mode = hibernation_ops ?
+ HIBERNATION_PLATFORM :
+ HIBERNATION_SHUTDOWN;
+ } else {
+ /* Restore swap signature. */
+ error = swsusp_unmark();
+ if (error)
+ pr_err("Swap will be unusable! Try swapon -a.\n");
+
+ return;
+ }
+ }
#endif
switch (hibernation_mode) {
if (pm_power_off)
kernel_power_off();
break;
-#ifdef CONFIG_SUSPEND
- case HIBERNATION_SUSPEND:
- error = suspend_devices_and_enter(PM_SUSPEND_MEM);
- if (error) {
- if (hibernation_ops)
- hibernation_mode = HIBERNATION_PLATFORM;
- else
- hibernation_mode = HIBERNATION_SHUTDOWN;
- power_down();
- }
- /*
- * Restore swap signature.
- */
- error = swsusp_unmark();
- if (error)
- printk(KERN_ERR "PM: Swap will be unusable! "
- "Try swapon -a.\n");
- return;
-#endif
}
kernel_halt();
/*
* Valid image is on the disk, if we continue we risk serious data
* corruption after resume.
*/
- printk(KERN_CRIT "PM: Please power down manually\n");
+ pr_crit("Power down manually\n");
while (1)
cpu_relax();
}
int error;
unsigned int flags;
- pr_debug("PM: Loading hibernation image.\n");
+ pr_debug("Loading hibernation image.\n");
lock_device_hotplug();
error = create_basic_memory_bitmaps();
if (!error)
hibernation_restore(flags & SF_PLATFORM_MODE);
- printk(KERN_ERR "PM: Failed to load hibernation image, recovering.\n");
+ pr_err("Failed to load hibernation image, recovering.\n");
swsusp_free();
free_basic_memory_bitmaps();
Unlock:
bool snapshot_test = false;
if (!hibernation_available()) {
- pr_debug("PM: Hibernation not available.\n");
+ pr_debug("Hibernation not available.\n");
return -EPERM;
}
goto Exit;
}
- printk(KERN_INFO "PM: Syncing filesystems ... ");
+ pr_info("Syncing filesystems ... \n");
sys_sync();
- printk("done.\n");
+ pr_info("done.\n");
error = freeze_processes();
if (error)
else
flags |= SF_CRC32_MODE;
- pr_debug("PM: writing image.\n");
+ pr_debug("Writing image.\n");
error = swsusp_write(flags);
swsusp_free();
if (!error) {
in_suspend = 0;
pm_restore_gfp_mask();
} else {
- pr_debug("PM: Image restored successfully.\n");
+ pr_debug("Image restored successfully.\n");
}
Free_bitmaps:
Thaw:
unlock_device_hotplug();
if (snapshot_test) {
- pr_debug("PM: Checking hibernation image\n");
+ pr_debug("Checking hibernation image\n");
error = swsusp_check();
if (!error)
error = load_image_and_restore();
goto Unlock;
}
- pr_debug("PM: Checking hibernation image partition %s\n", resume_file);
+ pr_debug("Checking hibernation image partition %s\n", resume_file);
if (resume_delay) {
- printk(KERN_INFO "Waiting %dsec before reading resume device...\n",
+ pr_info("Waiting %dsec before reading resume device ...\n",
resume_delay);
ssleep(resume_delay);
}
}
Check_image:
- pr_debug("PM: Hibernation image partition %d:%d present\n",
+ pr_debug("Hibernation image partition %d:%d present\n",
MAJOR(swsusp_resume_device), MINOR(swsusp_resume_device));
- pr_debug("PM: Looking for hibernation image.\n");
+ pr_debug("Looking for hibernation image.\n");
error = swsusp_check();
if (error)
goto Unlock;
goto Close_Finish;
}
- pr_debug("PM: Preparing processes for restore.\n");
+ pr_debug("Preparing processes for restore.\n");
error = freeze_processes();
if (error)
goto Close_Finish;
/* For success case, the suspend path will release the lock */
Unlock:
mutex_unlock(&pm_mutex);
- pr_debug("PM: Hibernation image not present or could not be loaded.\n");
+ pr_debug("Hibernation image not present or could not be loaded.\n");
return error;
Close_Finish:
swsusp_close(FMODE_READ);
error = -EINVAL;
if (!error)
- pr_debug("PM: Hibernation mode set to '%s'\n",
+ pr_debug("Hibernation mode set to '%s'\n",
hibernation_modes[mode]);
unlock_system_sleep();
return error ? error : n;
lock_system_sleep();
swsusp_resume_device = res;
unlock_system_sleep();
- printk(KERN_INFO "PM: Starting manual resume from disk\n");
+ pr_info("Starting manual resume from disk\n");
noresume = 0;
software_resume();
return n;
#include <linux/oom.h>
#include <linux/suspend.h>
#include <linux/module.h>
+#include <linux/sched/debug.h>
+#include <linux/sched/task.h>
#include <linux/syscalls.h>
#include <linux/freezer.h>
#include <linux/delay.h>
#include <linux/device.h>
#include <linux/init.h>
#include <linux/bootmem.h>
+#include <linux/nmi.h>
#include <linux/syscalls.h>
#include <linux/console.h>
#include <linux/highmem.h>
#include <linux/utsname.h>
#include <linux/ctype.h>
#include <linux/uio.h>
+#include <linux/sched/clock.h>
+#include <linux/sched/debug.h>
+#include <linux/sched/task_stack.h>
#include <linux/uaccess.h>
#include <asm/sections.h>
#include <linux/mutex.h>
#include <linux/slab.h>
#include <linux/vmalloc.h>
+#include <linux/sched/stat.h>
+
#include <asm/sections.h>
#include <asm/irq_regs.h>
#include <asm/ptrace.h>
#include <linux/capability.h>
#include <linux/export.h>
#include <linux/sched.h>
+#include <linux/sched/mm.h>
+#include <linux/sched/coredump.h>
+#include <linux/sched/task.h>
#include <linux/errno.h>
#include <linux/mm.h>
#include <linux/highmem.h>
#include <linux/rcupdate.h>
#include <linux/interrupt.h>
#include <linux/sched.h>
+#include <uapi/linux/sched/types.h>
#include <linux/atomic.h>
#include <linux/bitops.h>
#include <linux/completion.h>
#include <linux/smp.h>
#include <linux/rcupdate.h>
#include <linux/interrupt.h>
-#include <linux/sched.h>
+#include <linux/sched/signal.h>
+#include <uapi/linux/sched/types.h>
#include <linux/atomic.h>
#include <linux/bitops.h>
#include <linux/completion.h>
#include <linux/mutex.h>
#include <linux/percpu.h>
#include <linux/preempt.h>
-#include <linux/rcupdate.h>
+#include <linux/rcupdate_wait.h>
#include <linux/sched.h>
#include <linux/smp.h>
#include <linux/delay.h>
#include <linux/completion.h>
#include <linux/interrupt.h>
#include <linux/notifier.h>
-#include <linux/rcupdate.h>
+#include <linux/rcupdate_wait.h>
#include <linux/kernel.h>
#include <linux/export.h>
#include <linux/mutex.h>
#include "tiny_plugin.h"
+void rcu_barrier_bh(void)
+{
+ wait_rcu_gp(call_rcu_bh);
+}
+EXPORT_SYMBOL(rcu_barrier_bh);
+
+void rcu_barrier_sched(void)
+{
+ wait_rcu_gp(call_rcu_sched);
+}
+EXPORT_SYMBOL(rcu_barrier_sched);
+
#if defined(CONFIG_DEBUG_LOCK_ALLOC) || defined(CONFIG_RCU_TRACE)
/*
#include <linux/init.h>
#include <linux/spinlock.h>
#include <linux/smp.h>
-#include <linux/rcupdate.h>
+#include <linux/rcupdate_wait.h>
#include <linux/interrupt.h>
#include <linux/sched.h>
+#include <linux/sched/debug.h>
#include <linux/nmi.h>
#include <linux/atomic.h>
#include <linux/bitops.h>
#include <linux/kernel_stat.h>
#include <linux/wait.h>
#include <linux/kthread.h>
+#include <uapi/linux/sched/types.h>
#include <linux/prefetch.h>
#include <linux/delay.h>
#include <linux/stop_machine.h>
#include <linux/cache.h>
#include <linux/spinlock.h>
+#include <linux/rtmutex.h>
#include <linux/threads.h>
#include <linux/cpumask.h>
#include <linux/seqlock.h>
#include <linux/delay.h>
#include <linux/gfp.h>
#include <linux/oom.h>
+#include <linux/sched/debug.h>
#include <linux/smpboot.h>
+#include <uapi/linux/sched/types.h>
#include "../time/tick-internal.h"
#ifdef CONFIG_RCU_BOOST
#include <linux/spinlock.h>
#include <linux/smp.h>
#include <linux/interrupt.h>
-#include <linux/sched.h>
+#include <linux/sched/signal.h>
+#include <linux/sched/debug.h>
#include <linux/atomic.h>
#include <linux/bitops.h>
#include <linux/percpu.h>
#include <linux/moduleparam.h>
#include <linux/kthread.h>
#include <linux/tick.h>
+#include <linux/rcupdate_wait.h>
#define CREATE_TRACE_POINTS
#include <linux/kref.h>
#include <linux/rwsem.h>
+#include <linux/sched/autogroup.h>
struct autogroup {
/*
#include <linux/percpu.h>
#include <linux/ktime.h>
#include <linux/sched.h>
+#include <linux/nmi.h>
+#include <linux/sched/clock.h>
#include <linux/static_key.h>
#include <linux/workqueue.h>
#include <linux/compiler.h>
* Waiting for completion is a typically sync point, but not an exclusion point.
*/
-#include <linux/sched.h>
+#include <linux/sched/signal.h>
+#include <linux/sched/debug.h>
#include <linux/completion.h>
/**
* Copyright (C) 1991-2002 Linus Torvalds
*/
#include <linux/sched.h>
+#include <linux/sched/clock.h>
+#include <uapi/linux/sched/types.h>
+#include <linux/sched/loadavg.h>
+#include <linux/sched/hotplug.h>
#include <linux/cpuset.h>
#include <linux/delayacct.h>
#include <linux/init_task.h>
#include <linux/context_tracking.h>
+#include <linux/rcupdate_wait.h>
#include <linux/blkdev.h>
#include <linux/kprobes.h>
return rq;
/* Affinity changed (again). */
- if (!cpumask_test_cpu(dest_cpu, tsk_cpus_allowed(p)))
+ if (!cpumask_test_cpu(dest_cpu, &p->cpus_allowed))
return rq;
rq = move_queued_task(rq, p, dest_cpu);
if (task_cpu(arg->src_task) != arg->src_cpu)
goto unlock;
- if (!cpumask_test_cpu(arg->dst_cpu, tsk_cpus_allowed(arg->src_task)))
+ if (!cpumask_test_cpu(arg->dst_cpu, &arg->src_task->cpus_allowed))
goto unlock;
- if (!cpumask_test_cpu(arg->src_cpu, tsk_cpus_allowed(arg->dst_task)))
+ if (!cpumask_test_cpu(arg->src_cpu, &arg->dst_task->cpus_allowed))
goto unlock;
__migrate_swap_task(arg->src_task, arg->dst_cpu);
if (!cpu_active(arg.src_cpu) || !cpu_active(arg.dst_cpu))
goto out;
- if (!cpumask_test_cpu(arg.dst_cpu, tsk_cpus_allowed(arg.src_task)))
+ if (!cpumask_test_cpu(arg.dst_cpu, &arg.src_task->cpus_allowed))
goto out;
- if (!cpumask_test_cpu(arg.src_cpu, tsk_cpus_allowed(arg.dst_task)))
+ if (!cpumask_test_cpu(arg.src_cpu, &arg.dst_task->cpus_allowed))
goto out;
trace_sched_swap_numa(cur, arg.src_cpu, p, arg.dst_cpu);
for_each_cpu(dest_cpu, nodemask) {
if (!cpu_active(dest_cpu))
continue;
- if (cpumask_test_cpu(dest_cpu, tsk_cpus_allowed(p)))
+ if (cpumask_test_cpu(dest_cpu, &p->cpus_allowed))
return dest_cpu;
}
}
for (;;) {
/* Any allowed, online CPU? */
- for_each_cpu(dest_cpu, tsk_cpus_allowed(p)) {
+ for_each_cpu(dest_cpu, &p->cpus_allowed) {
if (!(p->flags & PF_KTHREAD) && !cpu_active(dest_cpu))
continue;
if (!cpu_online(dest_cpu))
{
lockdep_assert_held(&p->pi_lock);
- if (tsk_nr_cpus_allowed(p) > 1)
+ if (p->nr_cpus_allowed > 1)
cpu = p->sched_class->select_task_rq(p, cpu, sd_flags, wake_flags);
else
- cpu = cpumask_any(tsk_cpus_allowed(p));
+ cpu = cpumask_any(&p->cpus_allowed);
/*
* In order not to call set_task_cpu() on a blocking task we need
* [ this allows ->select_task() to simply return task_cpu(p) and
* not worry about this generic constraint ]
*/
- if (unlikely(!cpumask_test_cpu(cpu, tsk_cpus_allowed(p)) ||
+ if (unlikely(!cpumask_test_cpu(cpu, &p->cpus_allowed) ||
!cpu_online(cpu)))
cpu = select_fallback_rq(task_cpu(p), p);
static inline void preempt_latency_stop(int val) { }
#endif
+static inline unsigned long get_preempt_disable_ip(struct task_struct *p)
+{
+#ifdef CONFIG_DEBUG_PREEMPT
+ return p->preempt_disable_ip;
+#else
+ return 0;
+#endif
+}
+
/*
* Print scheduling while atomic bug:
*/
struct task_struct *p;
/*
- * Optimization: we know that if all tasks are in
- * the fair class we can call that function directly:
+ * Optimization: we know that if all tasks are in the fair class we can
+ * call that function directly, but only if the @prev task wasn't of a
+ * higher scheduling class, because otherwise those loose the
+ * opportunity to pull in more work from other CPUs.
*/
- if (likely(rq->nr_running == rq->cfs.h_nr_running)) {
+ if (likely((prev->sched_class == &idle_sched_class ||
+ prev->sched_class == &fair_sched_class) &&
+ rq->nr_running == rq->cfs.h_nr_running)) {
+
p = fair_sched_class.pick_next_task(rq, prev, rf);
if (unlikely(p == RETRY_TASK))
goto again;
int ppid;
unsigned long state = p->state;
+ /* Make sure the string lines up properly with the number of task states: */
+ BUILD_BUG_ON(sizeof(TASK_STATE_TO_CHAR_STR)-1 != ilog2(TASK_STATE_MAX)+1);
+
if (!try_get_task_stack(p))
return;
if (state)
if (curr_cpu == target_cpu)
return 0;
- if (!cpumask_test_cpu(target_cpu, tsk_cpus_allowed(p)))
+ if (!cpumask_test_cpu(target_cpu, &p->cpus_allowed))
return -EINVAL;
/* TODO: This is not properly updating schedstats */
const struct sched_dl_entity *dl_se = &p->dl;
if (later_mask &&
- cpumask_and(later_mask, cp->free_cpus, tsk_cpus_allowed(p))) {
+ cpumask_and(later_mask, cp->free_cpus, &p->cpus_allowed)) {
best_cpu = cpumask_any(later_mask);
goto out;
- } else if (cpumask_test_cpu(cpudl_maximum(cp), tsk_cpus_allowed(p)) &&
+ } else if (cpumask_test_cpu(cpudl_maximum(cp), &p->cpus_allowed) &&
dl_time_before(dl_se->deadline, cp->elements[0].dl)) {
best_cpu = cpudl_maximum(cp);
if (later_mask)
#include <linux/cpufreq.h>
#include <linux/kthread.h>
+#include <uapi/linux/sched/types.h>
#include <linux/slab.h>
#include <trace/events/power.h>
u64 last_freq_update_time;
s64 freq_update_delay_ns;
unsigned int next_freq;
+ unsigned int cached_raw_freq;
/* The next fields are only needed if fast switch cannot be used. */
struct irq_work irq_work;
struct update_util_data update_util;
struct sugov_policy *sg_policy;
- unsigned int cached_raw_freq;
unsigned long iowait_boost;
unsigned long iowait_boost_max;
u64 last_update;
/**
* get_next_freq - Compute a new frequency for a given cpufreq policy.
- * @sg_cpu: schedutil cpu object to compute the new frequency for.
+ * @sg_policy: schedutil policy object to compute the new frequency for.
* @util: Current CPU utilization.
* @max: CPU capacity.
*
* next_freq (as calculated above) is returned, subject to policy min/max and
* cpufreq driver limitations.
*/
-static unsigned int get_next_freq(struct sugov_cpu *sg_cpu, unsigned long util,
- unsigned long max)
+static unsigned int get_next_freq(struct sugov_policy *sg_policy,
+ unsigned long util, unsigned long max)
{
- struct sugov_policy *sg_policy = sg_cpu->sg_policy;
struct cpufreq_policy *policy = sg_policy->policy;
unsigned int freq = arch_scale_freq_invariant() ?
policy->cpuinfo.max_freq : policy->cur;
freq = (freq + (freq >> 2)) * util / max;
- if (freq == sg_cpu->cached_raw_freq && sg_policy->next_freq != UINT_MAX)
+ if (freq == sg_policy->cached_raw_freq && sg_policy->next_freq != UINT_MAX)
return sg_policy->next_freq;
- sg_cpu->cached_raw_freq = freq;
+ sg_policy->cached_raw_freq = freq;
return cpufreq_driver_resolve_freq(policy, freq);
}
} else {
sugov_get_util(&util, &max);
sugov_iowait_boost(sg_cpu, &util, &max);
- next_f = get_next_freq(sg_cpu, util, max);
+ next_f = get_next_freq(sg_policy, util, max);
}
sugov_update_commit(sg_policy, time, next_f);
}
sugov_iowait_boost(j_sg_cpu, &util, &max);
}
- return get_next_freq(sg_cpu, util, max);
+ return get_next_freq(sg_policy, util, max);
}
static void sugov_update_shared(struct update_util_data *hook, u64 time,
sg_policy->next_freq = UINT_MAX;
sg_policy->work_in_progress = false;
sg_policy->need_freq_update = false;
+ sg_policy->cached_raw_freq = 0;
for_each_cpu(cpu, policy->cpus) {
struct sugov_cpu *sg_cpu = &per_cpu(sugov_cpu, cpu);
sg_cpu->max = 0;
sg_cpu->flags = SCHED_CPUFREQ_RT;
sg_cpu->last_update = 0;
- sg_cpu->cached_raw_freq = 0;
sg_cpu->iowait_boost = 0;
sg_cpu->iowait_boost_max = policy->cpuinfo.max_freq;
cpufreq_add_update_util_hook(cpu, &sg_cpu->update_util,
if (skip)
continue;
- if (cpumask_any_and(tsk_cpus_allowed(p), vec->mask) >= nr_cpu_ids)
+ if (cpumask_any_and(&p->cpus_allowed, vec->mask) >= nr_cpu_ids)
continue;
if (lowest_mask) {
- cpumask_and(lowest_mask, tsk_cpus_allowed(p), vec->mask);
+ cpumask_and(lowest_mask, &p->cpus_allowed, vec->mask);
/*
* We have to ensure that we have at least one bit
#include <linux/kernel_stat.h>
#include <linux/static_key.h>
#include <linux/context_tracking.h>
-#include <linux/cputime.h>
+#include <linux/sched/cputime.h>
#include "sched.h"
-#ifdef CONFIG_PARAVIRT
-#include <asm/paravirt.h>
-#endif
-
#ifdef CONFIG_IRQ_TIME_ACCOUNTING
{
struct task_struct *p = dl_task_of(dl_se);
- if (tsk_nr_cpus_allowed(p) > 1)
+ if (p->nr_cpus_allowed > 1)
dl_rq->dl_nr_migratory++;
update_dl_migration(dl_rq);
{
struct task_struct *p = dl_task_of(dl_se);
- if (tsk_nr_cpus_allowed(p) > 1)
+ if (p->nr_cpus_allowed > 1)
dl_rq->dl_nr_migratory--;
update_dl_migration(dl_rq);
* If we cannot preempt any rq, fall back to pick any
* online cpu.
*/
- cpu = cpumask_any_and(cpu_active_mask, tsk_cpus_allowed(p));
+ cpu = cpumask_any_and(cpu_active_mask, &p->cpus_allowed);
if (cpu >= nr_cpu_ids) {
/*
* Fail to find any suitable cpu.
enqueue_dl_entity(&p->dl, pi_se, flags);
- if (!task_current(rq, p) && tsk_nr_cpus_allowed(p) > 1)
+ if (!task_current(rq, p) && p->nr_cpus_allowed > 1)
enqueue_pushable_dl_task(rq, p);
}
* try to make it stay here, it might be important.
*/
if (unlikely(dl_task(curr)) &&
- (tsk_nr_cpus_allowed(curr) < 2 ||
+ (curr->nr_cpus_allowed < 2 ||
!dl_entity_preempt(&p->dl, &curr->dl)) &&
- (tsk_nr_cpus_allowed(p) > 1)) {
+ (p->nr_cpus_allowed > 1)) {
int target = find_later_rq(p);
if (target != -1 &&
* Current can't be migrated, useless to reschedule,
* let's hope p can move out.
*/
- if (tsk_nr_cpus_allowed(rq->curr) == 1 ||
+ if (rq->curr->nr_cpus_allowed == 1 ||
cpudl_find(&rq->rd->cpudl, rq->curr, NULL) == -1)
return;
* p is migratable, so let's not schedule it and
* see if it is pushed or pulled somewhere else.
*/
- if (tsk_nr_cpus_allowed(p) != 1 &&
+ if (p->nr_cpus_allowed != 1 &&
cpudl_find(&rq->rd->cpudl, p, NULL) != -1)
return;
{
update_curr_dl(rq);
- if (on_dl_rq(&p->dl) && tsk_nr_cpus_allowed(p) > 1)
+ if (on_dl_rq(&p->dl) && p->nr_cpus_allowed > 1)
enqueue_pushable_dl_task(rq, p);
}
static int pick_dl_task(struct rq *rq, struct task_struct *p, int cpu)
{
if (!task_running(rq, p) &&
- cpumask_test_cpu(cpu, tsk_cpus_allowed(p)))
+ cpumask_test_cpu(cpu, &p->cpus_allowed))
return 1;
return 0;
}
if (unlikely(!later_mask))
return -1;
- if (tsk_nr_cpus_allowed(task) == 1)
+ if (task->nr_cpus_allowed == 1)
return -1;
/*
/* Retry if something changed. */
if (double_lock_balance(rq, later_rq)) {
if (unlikely(task_rq(task) != rq ||
- !cpumask_test_cpu(later_rq->cpu,
- tsk_cpus_allowed(task)) ||
+ !cpumask_test_cpu(later_rq->cpu, &task->cpus_allowed) ||
task_running(rq, task) ||
!dl_task(task) ||
!task_on_rq_queued(task))) {
BUG_ON(rq->cpu != task_cpu(p));
BUG_ON(task_current(rq, p));
- BUG_ON(tsk_nr_cpus_allowed(p) <= 1);
+ BUG_ON(p->nr_cpus_allowed <= 1);
BUG_ON(!task_on_rq_queued(p));
BUG_ON(!dl_task(p));
*/
if (dl_task(rq->curr) &&
dl_time_before(next_task->dl.deadline, rq->curr->dl.deadline) &&
- tsk_nr_cpus_allowed(rq->curr) > 1) {
+ rq->curr->nr_cpus_allowed > 1) {
resched_curr(rq);
return 0;
}
{
if (!task_running(rq, p) &&
!test_tsk_need_resched(rq->curr) &&
- tsk_nr_cpus_allowed(p) > 1 &&
+ p->nr_cpus_allowed > 1 &&
dl_task(rq->curr) &&
- (tsk_nr_cpus_allowed(rq->curr) < 2 ||
+ (rq->curr->nr_cpus_allowed < 2 ||
!dl_entity_preempt(&p->dl, &rq->curr->dl))) {
push_dl_tasks(rq);
}
if (rq->curr != p) {
#ifdef CONFIG_SMP
- if (tsk_nr_cpus_allowed(p) > 1 && rq->dl.overloaded)
+ if (p->nr_cpus_allowed > 1 && rq->dl.overloaded)
queue_push_tasks(rq);
#endif
if (dl_task(rq->curr))
*/
#include <linux/proc_fs.h>
-#include <linux/sched.h>
+#include <linux/sched/mm.h>
+#include <linux/sched/task.h>
#include <linux/seq_file.h>
#include <linux/kallsyms.h>
#include <linux/utsname.h>
* Copyright (C) 2007 Red Hat, Inc., Peter Zijlstra
*/
-#include <linux/sched.h>
+#include <linux/sched/mm.h>
+#include <linux/sched/topology.h>
+
#include <linux/latencytop.h>
#include <linux/cpumask.h>
#include <linux/cpuidle.h>
*/
if (cur) {
/* Skip this swap candidate if cannot move to the source cpu */
- if (!cpumask_test_cpu(env->src_cpu, tsk_cpus_allowed(cur)))
+ if (!cpumask_test_cpu(env->src_cpu, &cur->cpus_allowed))
goto unlock;
/*
for_each_cpu(cpu, cpumask_of_node(env->dst_nid)) {
/* Skip this CPU if the source task cannot migrate */
- if (!cpumask_test_cpu(cpu, tsk_cpus_allowed(env->p)))
+ if (!cpumask_test_cpu(cpu, &env->p->cpus_allowed))
continue;
env->dst_cpu = cpu;
/* Skip over this group if it has no CPUs allowed */
if (!cpumask_intersects(sched_group_cpus(group),
- tsk_cpus_allowed(p)))
+ &p->cpus_allowed))
continue;
local_group = cpumask_test_cpu(this_cpu,
return cpumask_first(sched_group_cpus(group));
/* Traverse only the allowed CPUs */
- for_each_cpu_and(i, sched_group_cpus(group), tsk_cpus_allowed(p)) {
+ for_each_cpu_and(i, sched_group_cpus(group), &p->cpus_allowed) {
if (idle_cpu(i)) {
struct rq *rq = cpu_rq(i);
struct cpuidle_state *idle = idle_get_state(rq);
if (!test_idle_cores(target, false))
return -1;
- cpumask_and(cpus, sched_domain_span(sd), tsk_cpus_allowed(p));
+ cpumask_and(cpus, sched_domain_span(sd), &p->cpus_allowed);
for_each_cpu_wrap(core, cpus, target, wrap) {
bool idle = true;
return -1;
for_each_cpu(cpu, cpu_smt_mask(target)) {
- if (!cpumask_test_cpu(cpu, tsk_cpus_allowed(p)))
+ if (!cpumask_test_cpu(cpu, &p->cpus_allowed))
continue;
if (idle_cpu(cpu))
return cpu;
* Due to large variance we need a large fuzz factor; hackbench in
* particularly is sensitive here.
*/
- if ((avg_idle / 512) < avg_cost)
+ if (sched_feat(SIS_AVG_CPU) && (avg_idle / 512) < avg_cost)
return -1;
time = local_clock();
for_each_cpu_wrap(cpu, sched_domain_span(sd), target, wrap) {
- if (!cpumask_test_cpu(cpu, tsk_cpus_allowed(p)))
+ if (!cpumask_test_cpu(cpu, &p->cpus_allowed))
continue;
if (idle_cpu(cpu))
break;
if (sd_flag & SD_BALANCE_WAKE) {
record_wakee(p);
want_affine = !wake_wide(p) && !wake_cap(p, cpu, prev_cpu)
- && cpumask_test_cpu(cpu, tsk_cpus_allowed(p));
+ && cpumask_test_cpu(cpu, &p->cpus_allowed);
}
rcu_read_lock();
if (throttled_lb_pair(task_group(p), env->src_cpu, env->dst_cpu))
return 0;
- if (!cpumask_test_cpu(env->dst_cpu, tsk_cpus_allowed(p))) {
+ if (!cpumask_test_cpu(env->dst_cpu, &p->cpus_allowed)) {
int cpu;
schedstat_inc(p->se.statistics.nr_failed_migrations_affine);
/* Prevent to re-select dst_cpu via env's cpus */
for_each_cpu_and(cpu, env->dst_grpmask, env->cpus) {
- if (cpumask_test_cpu(cpu, tsk_cpus_allowed(p))) {
+ if (cpumask_test_cpu(cpu, &p->cpus_allowed)) {
env->flags |= LBF_DST_PINNED;
env->new_dst_cpu = cpu;
break;
/*
* Group imbalance indicates (and tries to solve) the problem where balancing
- * groups is inadequate due to tsk_cpus_allowed() constraints.
+ * groups is inadequate due to ->cpus_allowed constraints.
*
* Imagine a situation of two groups of 4 cpus each and 4 tasks each with a
* cpumask covering 1 cpu of the first group and 3 cpus of the second group.
* if the curr task on busiest cpu can't be
* moved to this_cpu
*/
- if (!cpumask_test_cpu(this_cpu,
- tsk_cpus_allowed(busiest->curr))) {
+ if (!cpumask_test_cpu(this_cpu, &busiest->curr->cpus_allowed)) {
raw_spin_unlock_irqrestore(&busiest->lock,
flags);
env.flags |= LBF_ALL_PINNED;
*/
SCHED_FEAT(TTWU_QUEUE, true)
+/*
+ * When doing wakeups, attempt to limit superfluous scans of the LLC domain.
+ */
+SCHED_FEAT(SIS_AVG_CPU, false)
+
#ifdef HAVE_RT_PUSH_IPI
/*
* In order to avoid a thundering herd attack of CPUs that are
* Generic entry point for the idle threads
*/
#include <linux/sched.h>
+#include <linux/sched/idle.h>
#include <linux/cpu.h>
#include <linux/cpuidle.h>
#include <linux/cpuhotplug.h>
*/
#include <linux/export.h>
+#include <linux/sched/loadavg.h>
#include "sched.h"
rt_rq = &rq_of_rt_rq(rt_rq)->rt;
rt_rq->rt_nr_total++;
- if (tsk_nr_cpus_allowed(p) > 1)
+ if (p->nr_cpus_allowed > 1)
rt_rq->rt_nr_migratory++;
update_rt_migration(rt_rq);
rt_rq = &rq_of_rt_rq(rt_rq)->rt;
rt_rq->rt_nr_total--;
- if (tsk_nr_cpus_allowed(p) > 1)
+ if (p->nr_cpus_allowed > 1)
rt_rq->rt_nr_migratory--;
update_rt_migration(rt_rq);
enqueue_rt_entity(rt_se, flags);
- if (!task_current(rq, p) && tsk_nr_cpus_allowed(p) > 1)
+ if (!task_current(rq, p) && p->nr_cpus_allowed > 1)
enqueue_pushable_task(rq, p);
}
* will have to sort it out.
*/
if (curr && unlikely(rt_task(curr)) &&
- (tsk_nr_cpus_allowed(curr) < 2 ||
+ (curr->nr_cpus_allowed < 2 ||
curr->prio <= p->prio)) {
int target = find_lowest_rq(p);
* Current can't be migrated, useless to reschedule,
* let's hope p can move out.
*/
- if (tsk_nr_cpus_allowed(rq->curr) == 1 ||
+ if (rq->curr->nr_cpus_allowed == 1 ||
!cpupri_find(&rq->rd->cpupri, rq->curr, NULL))
return;
* p is migratable, so let's not schedule it and
* see if it is pushed or pulled somewhere else.
*/
- if (tsk_nr_cpus_allowed(p) != 1
+ if (p->nr_cpus_allowed != 1
&& cpupri_find(&rq->rd->cpupri, p, NULL))
return;
* The previous task needs to be made eligible for pushing
* if it is still active
*/
- if (on_rt_rq(&p->rt) && tsk_nr_cpus_allowed(p) > 1)
+ if (on_rt_rq(&p->rt) && p->nr_cpus_allowed > 1)
enqueue_pushable_task(rq, p);
}
static int pick_rt_task(struct rq *rq, struct task_struct *p, int cpu)
{
if (!task_running(rq, p) &&
- cpumask_test_cpu(cpu, tsk_cpus_allowed(p)))
+ cpumask_test_cpu(cpu, &p->cpus_allowed))
return 1;
return 0;
}
if (unlikely(!lowest_mask))
return -1;
- if (tsk_nr_cpus_allowed(task) == 1)
+ if (task->nr_cpus_allowed == 1)
return -1; /* No other targets possible */
if (!cpupri_find(&task_rq(task)->rd->cpupri, task, lowest_mask))
* Also make sure that it wasn't scheduled on its rq.
*/
if (unlikely(task_rq(task) != rq ||
- !cpumask_test_cpu(lowest_rq->cpu,
- tsk_cpus_allowed(task)) ||
+ !cpumask_test_cpu(lowest_rq->cpu, &task->cpus_allowed) ||
task_running(rq, task) ||
!rt_task(task) ||
!task_on_rq_queued(task))) {
BUG_ON(rq->cpu != task_cpu(p));
BUG_ON(task_current(rq, p));
- BUG_ON(tsk_nr_cpus_allowed(p) <= 1);
+ BUG_ON(p->nr_cpus_allowed <= 1);
BUG_ON(!task_on_rq_queued(p));
BUG_ON(!rt_task(p));
{
if (!task_running(rq, p) &&
!test_tsk_need_resched(rq->curr) &&
- tsk_nr_cpus_allowed(p) > 1 &&
+ p->nr_cpus_allowed > 1 &&
(dl_task(rq->curr) || rt_task(rq->curr)) &&
- (tsk_nr_cpus_allowed(rq->curr) < 2 ||
+ (rq->curr->nr_cpus_allowed < 2 ||
rq->curr->prio <= p->prio))
push_rt_tasks(rq);
}
*/
if (task_on_rq_queued(p) && rq->curr != p) {
#ifdef CONFIG_SMP
- if (tsk_nr_cpus_allowed(p) > 1 && rq->rt.overloaded)
+ if (p->nr_cpus_allowed > 1 && rq->rt.overloaded)
queue_push_tasks(rq);
#endif /* CONFIG_SMP */
if (p->prio < rq->curr->prio)
#include <linux/sched.h>
+#include <linux/sched/autogroup.h>
#include <linux/sched/sysctl.h>
+#include <linux/sched/topology.h>
#include <linux/sched/rt.h>
-#include <linux/u64_stats_sync.h>
#include <linux/sched/deadline.h>
+#include <linux/sched/clock.h>
+#include <linux/sched/wake_q.h>
+#include <linux/sched/signal.h>
+#include <linux/sched/numa_balancing.h>
+#include <linux/sched/mm.h>
+#include <linux/sched/cpufreq.h>
+#include <linux/sched/stat.h>
+#include <linux/sched/nohz.h>
+#include <linux/sched/debug.h>
+#include <linux/sched/hotplug.h>
+#include <linux/sched/task.h>
+#include <linux/sched/task_stack.h>
+#include <linux/sched/cputime.h>
+#include <linux/sched/init.h>
+
+#include <linux/u64_stats_sync.h>
#include <linux/kernel_stat.h>
#include <linux/binfmts.h>
#include <linux/mutex.h>
#include <linux/tick.h>
#include <linux/slab.h>
+#ifdef CONFIG_PARAVIRT
+#include <asm/paravirt.h>
+#endif
+
#include "cpupri.h"
#include "cpudeadline.h"
#include "cpuacct.h"
extern void print_dl_stats(struct seq_file *m, int cpu);
extern void
print_cfs_rq(struct seq_file *m, int cpu, struct cfs_rq *cfs_rq);
-
#ifdef CONFIG_NUMA_BALANCING
extern void
show_numa_stats(struct task_struct *p, struct seq_file *m);
#define sched_info_arrive(rq, next) do { } while (0)
#define sched_info_switch(rq, t, next) do { } while (0)
#endif /* CONFIG_SCHED_INFO */
-
-/*
- * The following are functions that support scheduler-internal time accounting.
- * These functions are generally called at the timer tick. None of this depends
- * on CONFIG_SCHEDSTATS.
- */
-
-/**
- * get_running_cputimer - return &tsk->signal->cputimer if cputimer is running
- *
- * @tsk: Pointer to target task.
- */
-#ifdef CONFIG_POSIX_TIMERS
-static inline
-struct thread_group_cputimer *get_running_cputimer(struct task_struct *tsk)
-{
- struct thread_group_cputimer *cputimer = &tsk->signal->cputimer;
-
- /* Check if cputimer isn't running. This is accessed without locking. */
- if (!READ_ONCE(cputimer->running))
- return NULL;
-
- /*
- * After we flush the task's sum_exec_runtime to sig->sum_sched_runtime
- * in __exit_signal(), we won't account to the signal struct further
- * cputime consumed by that task, even though the task can still be
- * ticking after __exit_signal().
- *
- * In order to keep a consistent behaviour between thread group cputime
- * and thread group cputimer accounting, lets also ignore the cputime
- * elapsing after __exit_signal() in any thread group timer running.
- *
- * This makes sure that POSIX CPU clocks and timers are synchronized, so
- * that a POSIX CPU timer won't expire while the corresponding POSIX CPU
- * clock delta is behind the expiring timer value.
- */
- if (unlikely(!tsk->sighand))
- return NULL;
-
- return cputimer;
-}
-#else
-static inline
-struct thread_group_cputimer *get_running_cputimer(struct task_struct *tsk)
-{
- return NULL;
-}
-#endif
-
-/**
- * account_group_user_time - Maintain utime for a thread group.
- *
- * @tsk: Pointer to task structure.
- * @cputime: Time value by which to increment the utime field of the
- * thread_group_cputime structure.
- *
- * If thread group time is being maintained, get the structure for the
- * running CPU and update the utime field there.
- */
-static inline void account_group_user_time(struct task_struct *tsk,
- u64 cputime)
-{
- struct thread_group_cputimer *cputimer = get_running_cputimer(tsk);
-
- if (!cputimer)
- return;
-
- atomic64_add(cputime, &cputimer->cputime_atomic.utime);
-}
-
-/**
- * account_group_system_time - Maintain stime for a thread group.
- *
- * @tsk: Pointer to task structure.
- * @cputime: Time value by which to increment the stime field of the
- * thread_group_cputime structure.
- *
- * If thread group time is being maintained, get the structure for the
- * running CPU and update the stime field there.
- */
-static inline void account_group_system_time(struct task_struct *tsk,
- u64 cputime)
-{
- struct thread_group_cputimer *cputimer = get_running_cputimer(tsk);
-
- if (!cputimer)
- return;
-
- atomic64_add(cputime, &cputimer->cputime_atomic.stime);
-}
-
-/**
- * account_group_exec_runtime - Maintain exec runtime for a thread group.
- *
- * @tsk: Pointer to task structure.
- * @ns: Time value by which to increment the sum_exec_runtime field
- * of the thread_group_cputime structure.
- *
- * If thread group time is being maintained, get the structure for the
- * running CPU and update the sum_exec_runtime field there.
- */
-static inline void account_group_exec_runtime(struct task_struct *tsk,
- unsigned long long ns)
-{
- struct thread_group_cputimer *cputimer = get_running_cputimer(tsk);
-
- if (!cputimer)
- return;
-
- atomic64_add(ns, &cputimer->cputime_atomic.sum_exec_runtime);
-}
-#include <linux/sched.h>
+#include <linux/sched/signal.h>
#include <linux/swait.h>
void __init_swait_queue_head(struct swait_queue_head *q, const char *name,
*/
#include <linux/init.h>
#include <linux/export.h>
-#include <linux/sched.h>
+#include <linux/sched/signal.h>
+#include <linux/sched/debug.h>
#include <linux/mm.h>
#include <linux/wait.h>
#include <linux/hash.h>
}
EXPORT_SYMBOL(prepare_to_wait_event);
+/*
+ * Note! These two wait functions are entered with the
+ * wait-queue lock held (and interrupts off in the _irq
+ * case), so there is no race with testing the wakeup
+ * condition in the caller before they add the wait
+ * entry to the wake queue.
+ */
+int do_wait_intr(wait_queue_head_t *wq, wait_queue_t *wait)
+{
+ if (likely(list_empty(&wait->task_list)))
+ __add_wait_queue_tail(wq, wait);
+
+ set_current_state(TASK_INTERRUPTIBLE);
+ if (signal_pending(current))
+ return -ERESTARTSYS;
+
+ spin_unlock(&wq->lock);
+ schedule();
+ spin_lock(&wq->lock);
+ return 0;
+}
+EXPORT_SYMBOL(do_wait_intr);
+
+int do_wait_intr_irq(wait_queue_head_t *wq, wait_queue_t *wait)
+{
+ if (likely(list_empty(&wait->task_list)))
+ __add_wait_queue_tail(wq, wait);
+
+ set_current_state(TASK_INTERRUPTIBLE);
+ if (signal_pending(current))
+ return -ERESTARTSYS;
+
+ spin_unlock_irq(&wq->lock);
+ schedule();
+ spin_lock_irq(&wq->lock);
+ return 0;
+}
+EXPORT_SYMBOL(do_wait_intr_irq);
+
/**
* finish_wait - clean up after waiting in a queue
* @q: waitqueue waited on
#include <linux/compat.h>
#include <linux/coredump.h>
#include <linux/sched.h>
+#include <linux/sched/task_stack.h>
#include <linux/seccomp.h>
#include <linux/slab.h>
#include <linux/syscalls.h>
#include <linux/slab.h>
#include <linux/export.h>
#include <linux/init.h>
-#include <linux/sched.h>
+#include <linux/sched/mm.h>
+#include <linux/sched/user.h>
+#include <linux/sched/debug.h>
+#include <linux/sched/task.h>
+#include <linux/sched/task_stack.h>
+#include <linux/sched/cputime.h>
#include <linux/fs.h>
#include <linux/tty.h>
#include <linux/binfmts.h>
* @tsk is about to have PF_EXITING set - lock out users which
* expect stable threadgroup.
*/
- threadgroup_change_begin(tsk);
+ cgroup_threadgroup_change_begin(tsk);
if (thread_group_empty(tsk) || signal_group_exit(tsk->signal)) {
tsk->flags |= PF_EXITING;
- threadgroup_change_end(tsk);
+ cgroup_threadgroup_change_end(tsk);
return;
}
*/
tsk->flags |= PF_EXITING;
- threadgroup_change_end(tsk);
+ cgroup_threadgroup_change_end(tsk);
if (!signal_pending(tsk))
goto out;
#include <linux/smp.h>
#include <linux/cpu.h>
#include <linux/sched.h>
+#include <linux/sched/idle.h>
#include <linux/hypervisor.h>
#include "smpboot.h"
#include <linux/list.h>
#include <linux/slab.h>
#include <linux/sched.h>
+#include <linux/sched/task.h>
#include <linux/export.h>
#include <linux/percpu.h>
#include <linux/kthread.h>
#include <linux/binfmts.h>
#include <linux/sched.h>
+#include <linux/sched/autogroup.h>
+#include <linux/sched/loadavg.h>
+#include <linux/sched/stat.h>
+#include <linux/sched/mm.h>
+#include <linux/sched/coredump.h>
+#include <linux/sched/task.h>
+#include <linux/sched/cputime.h>
#include <linux/rcupdate.h>
#include <linux/uidgid.h>
#include <linux/cred.h>
#include <linux/capability.h>
#include <linux/binfmts.h>
#include <linux/sched/sysctl.h>
+#include <linux/sched/coredump.h>
#include <linux/kexec.h>
#include <linux/bpf.h>
#include <linux/mount.h>
#include <linux/hrtimer.h>
#include <linux/timerqueue.h>
#include <linux/rtc.h>
+#include <linux/sched/signal.h>
+#include <linux/sched/debug.h>
#include <linux/alarmtimer.h>
#include <linux/mutex.h>
#include <linux/platform_device.h>
#include <linux/seq_file.h>
#include <linux/err.h>
#include <linux/debugobjects.h>
-#include <linux/sched.h>
+#include <linux/sched/signal.h>
#include <linux/sched/sysctl.h>
#include <linux/sched/rt.h>
#include <linux/sched/deadline.h>
+#include <linux/sched/nohz.h>
+#include <linux/sched/debug.h>
#include <linux/timer.h>
#include <linux/freezer.h>
#include <linux/interrupt.h>
#include <linux/syscalls.h>
#include <linux/time.h>
+#include <linux/sched/signal.h>
+#include <linux/sched/cputime.h>
#include <linux/posix-timers.h>
#include <linux/hrtimer.h>
#include <trace/events/timer.h>
shift_hz += cycles_per_tick/2;
do_div(shift_hz, cycles_per_tick);
/* Calculate nsec_per_tick using shift_hz */
- nsec_per_tick = (u64)TICK_NSEC << 8;
+ nsec_per_tick = (u64)NSEC_PER_SEC << 8;
nsec_per_tick += (u32)shift_hz/2;
do_div(nsec_per_tick, (u32)shift_hz);
* Implement CPU time clocks for the POSIX clock interface.
*/
-#include <linux/sched.h>
+#include <linux/sched/signal.h>
+#include <linux/sched/cputime.h>
#include <linux/posix-timers.h>
#include <linux/errno.h>
#include <linux/math64.h>
#include <linux/slab.h>
#include <linux/time.h>
#include <linux/mutex.h>
+#include <linux/sched/task.h>
#include <linux/uaccess.h>
#include <linux/list.h>
#include <linux/kernel.h>
#include <linux/moduleparam.h>
#include <linux/sched.h>
+#include <linux/sched/clock.h>
#include <linux/syscore_ops.h>
#include <linux/hrtimer.h>
#include <linux/sched_clock.h>
#include <linux/interrupt.h>
#include <linux/kernel_stat.h>
#include <linux/percpu.h>
+#include <linux/nmi.h>
#include <linux/profile.h>
-#include <linux/sched.h>
+#include <linux/sched/signal.h>
+#include <linux/sched/clock.h>
+#include <linux/sched/stat.h>
+#include <linux/sched/nohz.h>
#include <linux/module.h>
#include <linux/irq_work.h>
#include <linux/posix-timers.h>
#include <linux/percpu.h>
#include <linux/init.h>
#include <linux/mm.h>
+#include <linux/nmi.h>
#include <linux/sched.h>
+#include <linux/sched/loadavg.h>
#include <linux/syscore_ops.h>
#include <linux/clocksource.h>
#include <linux/jiffies.h>
#include <linux/tick.h>
#include <linux/kallsyms.h>
#include <linux/irq_work.h>
-#include <linux/sched.h>
+#include <linux/sched/signal.h>
#include <linux/sched/sysctl.h>
+#include <linux/sched/nohz.h>
+#include <linux/sched/debug.h>
#include <linux/slab.h>
#include <linux/compat.h>
#include <linux/smp.h>
#include <linux/interrupt.h>
#include <linux/sched.h>
+#include <linux/sched/clock.h>
#include <linux/atomic.h>
#include <linux/bitops.h>
#include <linux/completion.h>
If unsure, say N.
-config KPROBE_EVENT
+config KPROBE_EVENTS
depends on KPROBES
depends on HAVE_REGS_AND_STACK_ACCESS_API
bool "Enable kprobes-based dynamic events"
This option is also required by perf-probe subcommand of perf tools.
If you want to use perf tools, this option is strongly recommended.
-config UPROBE_EVENT
+config UPROBE_EVENTS
bool "Enable uprobes-based dynamic events"
depends on ARCH_SUPPORTS_UPROBES
depends on MMU
config BPF_EVENTS
depends on BPF_SYSCALL
- depends on (KPROBE_EVENT || UPROBE_EVENT) && PERF_EVENTS
+ depends on (KPROBE_EVENTS || UPROBE_EVENTS) && PERF_EVENTS
bool
default y
help
obj-$(CONFIG_EVENT_TRACING) += trace_events_trigger.o
obj-$(CONFIG_HIST_TRIGGERS) += trace_events_hist.o
obj-$(CONFIG_BPF_EVENTS) += bpf_trace.o
-obj-$(CONFIG_KPROBE_EVENT) += trace_kprobe.o
+obj-$(CONFIG_KPROBE_EVENTS) += trace_kprobe.o
obj-$(CONFIG_TRACEPOINTS) += power-traces.o
ifeq ($(CONFIG_PM),y)
obj-$(CONFIG_TRACEPOINTS) += rpm-traces.o
obj-$(CONFIG_KGDB_KDB) += trace_kdb.o
endif
obj-$(CONFIG_PROBE_EVENTS) += trace_probe.o
-obj-$(CONFIG_UPROBE_EVENT) += trace_uprobe.o
+obj-$(CONFIG_UPROBE_EVENTS) += trace_uprobe.o
obj-$(CONFIG_TRACEPOINT_BENCHMARK) += trace_benchmark.o
#include <linux/stop_machine.h>
#include <linux/clocksource.h>
+#include <linux/sched/task.h>
#include <linux/kallsyms.h>
#include <linux/seq_file.h>
#include <linux/suspend.h>
}
__setup("ftrace_graph_notrace=", set_graph_notrace_function);
+static int __init set_graph_max_depth_function(char *str)
+{
+ if (!str)
+ return 0;
+ fgraph_max_depth = simple_strtoul(str, NULL, 0);
+ return 1;
+}
+__setup("ftrace_graph_max_depth=", set_graph_max_depth_function);
+
static void __init set_ftrace_early_graph(char *buf, int enable)
{
int ret;
char *func;
struct ftrace_hash *hash;
- if (enable)
- hash = ftrace_graph_hash;
- else
- hash = ftrace_graph_notrace_hash;
+ hash = alloc_ftrace_hash(FTRACE_HASH_DEFAULT_BITS);
+ if (WARN_ON(!hash))
+ return;
while (buf) {
func = strsep(&buf, ",");
printk(KERN_DEBUG "ftrace: function %s not "
"traceable\n", func);
}
+
+ if (enable)
+ ftrace_graph_hash = hash;
+ else
+ ftrace_graph_notrace_hash = hash;
}
#endif /* CONFIG_FUNCTION_GRAPH_TRACER */
* Normally the mcount trampoline will call the ops->func, but there
* are times that it should not. For example, if the ops does not
* have its own recursion protection, then it should call the
- * ftrace_ops_recurs_func() instead.
+ * ftrace_ops_assist_func() instead.
*
* Returns the function that the trampoline should call for @ops.
*/
#include <linux/trace_events.h>
#include <linux/ring_buffer.h>
#include <linux/trace_clock.h>
+#include <linux/sched/clock.h>
#include <linux/trace_seq.h>
#include <linux/spinlock.h>
#include <linux/irq_work.h>
#include <linux/ring_buffer.h>
#include <linux/completion.h>
#include <linux/kthread.h>
+#include <uapi/linux/sched/types.h>
#include <linux/module.h>
#include <linux/ktime.h>
#include <asm/local.h>
"\t\t\t traces\n"
#endif
#endif /* CONFIG_STACK_TRACER */
-#ifdef CONFIG_KPROBE_EVENT
+#ifdef CONFIG_KPROBE_EVENTS
" kprobe_events\t\t- Add/remove/show the kernel dynamic events\n"
"\t\t\t Write into this file to define/undefine new trace events.\n"
#endif
-#ifdef CONFIG_UPROBE_EVENT
+#ifdef CONFIG_UPROBE_EVENTS
" uprobe_events\t\t- Add/remove/show the userspace dynamic events\n"
"\t\t\t Write into this file to define/undefine new trace events.\n"
#endif
-#if defined(CONFIG_KPROBE_EVENT) || defined(CONFIG_UPROBE_EVENT)
+#if defined(CONFIG_KPROBE_EVENTS) || defined(CONFIG_UPROBE_EVENTS)
"\t accepts: event-definitions (one definition per line)\n"
"\t Format: p|r[:[<group>/]<event>] <place> [<args>]\n"
"\t -:[<group>/]<event>\n"
-#ifdef CONFIG_KPROBE_EVENT
+#ifdef CONFIG_KPROBE_EVENTS
"\t place: [<module>:]<symbol>[+<offset>]|<memaddr>\n"
#endif
-#ifdef CONFIG_UPROBE_EVENT
+#ifdef CONFIG_UPROBE_EVENTS
"\t place: <path>:<offset>\n"
#endif
"\t args: <name>=fetcharg[:type]\n"
#include <linux/module.h>
#include <linux/percpu.h>
#include <linux/sched.h>
+#include <linux/sched/clock.h>
#include <linux/ktime.h>
#include <linux/trace_clock.h>
#include <linux/mutex.h>
#include <linux/slab.h>
#include <linux/stacktrace.h>
+#include <linux/rculist.h>
#include "tracing_map.h"
#include "trace.h"
#include <linux/ctype.h>
#include <linux/mutex.h>
#include <linux/slab.h>
+#include <linux/rculist.h>
#include "trace.h"
#include <linux/uaccess.h>
#include <linux/cpumask.h>
#include <linux/delay.h>
+#include <linux/sched/clock.h>
#include "trace.h"
static struct trace_array *hwlat_trace;
#include <linux/module.h>
#include <linux/uaccess.h>
+#include <linux/rculist.h>
#include "trace_probe.h"
#include <linux/module.h>
#include <linux/mutex.h>
#include <linux/ftrace.h>
+#include <linux/sched/clock.h>
+#include <linux/sched/mm.h>
#include "trace_output.h"
#define FETCH_TYPE_STRING 0
#define FETCH_TYPE_STRSIZE 1
-#ifdef CONFIG_KPROBE_EVENT
+#ifdef CONFIG_KPROBE_EVENTS
struct symbol_cache;
unsigned long update_symbol_cache(struct symbol_cache *sc);
void free_symbol_cache(struct symbol_cache *sc);
{
return NULL;
}
-#endif /* CONFIG_KPROBE_EVENT */
+#endif /* CONFIG_KPROBE_EVENTS */
struct probe_arg {
struct fetch_param fetch;
/* Include in trace.c */
+#include <uapi/linux/sched/types.h>
#include <linux/stringify.h>
#include <linux/kthread.h>
#include <linux/delay.h>
* Copyright (C) 2008 Steven Rostedt <srostedt@redhat.com>
*
*/
+#include <linux/sched/task_stack.h>
#include <linux/stacktrace.h>
#include <linux/kallsyms.h>
#include <linux/seq_file.h>
}
/*
- * When arch-specific code overides this function, the following
+ * When arch-specific code overrides this function, the following
* data should be filled up, assuming stack_trace_max_lock is held to
* prevent concurrent updates.
* stack_trace_index[]
#include <linux/uprobes.h>
#include <linux/namei.h>
#include <linux/string.h>
+#include <linux/rculist.h>
#include "trace_probe.h"
#include <linux/tracepoint.h>
#include <linux/err.h>
#include <linux/slab.h>
-#include <linux/sched.h>
+#include <linux/sched/signal.h>
+#include <linux/sched/task.h>
#include <linux/static_key.h>
extern struct tracepoint * const __start___tracepoints_ptrs[];
*/
#include <linux/kernel.h>
-#include <linux/sched.h>
+#include <linux/sched/signal.h>
+#include <linux/sched/mm.h>
+#include <linux/sched/cputime.h>
#include <linux/tsacct_kern.h>
#include <linux/acct.h>
#include <linux/jiffies.h>
#include <linux/stat.h>
#include <linux/sysctl.h>
#include <linux/slab.h>
+#include <linux/cred.h>
#include <linux/hash.h>
#include <linux/user_namespace.h>
new->ns = ns;
new->uid = uid;
- atomic_set(&new->count, 0);
+ new->count = 0;
spin_lock_irq(&ucounts_lock);
ucounts = find_ucounts(ns, uid, hashent);
ucounts = new;
}
}
- if (!atomic_add_unless(&ucounts->count, 1, INT_MAX))
+ if (ucounts->count == INT_MAX)
ucounts = NULL;
+ else
+ ucounts->count += 1;
spin_unlock_irq(&ucounts_lock);
return ucounts;
}
{
unsigned long flags;
- if (atomic_dec_and_test(&ucounts->count)) {
- spin_lock_irqsave(&ucounts_lock, flags);
+ spin_lock_irqsave(&ucounts_lock, flags);
+ ucounts->count -= 1;
+ if (!ucounts->count)
hlist_del_init(&ucounts->node);
- spin_unlock_irqrestore(&ucounts_lock, flags);
+ else
+ ucounts = NULL;
+ spin_unlock_irqrestore(&ucounts_lock, flags);
- kfree(ucounts);
- }
+ kfree(ucounts);
}
static inline bool atomic_inc_below(atomic_t *v, int u)
#include <linux/init.h>
#include <linux/highuid.h>
#include <linux/security.h>
+#include <linux/cred.h>
#include <linux/syscalls.h>
#include <linux/uaccess.h>
#include <linux/slab.h>
#include <linux/bitops.h>
#include <linux/key.h>
+#include <linux/sched/user.h>
#include <linux/interrupt.h>
#include <linux/export.h>
#include <linux/user_namespace.h>
#include <linux/export.h>
#include <linux/nsproxy.h>
#include <linux/slab.h>
+#include <linux/sched/signal.h>
#include <linux/user_namespace.h>
#include <linux/proc_ns.h>
#include <linux/highuid.h>
#include <linux/utsname.h>
#include <linux/err.h>
#include <linux/slab.h>
+#include <linux/cred.h>
#include <linux/user_namespace.h>
#include <linux/proc_ns.h>
+#include <linux/sched/task.h>
static struct ucounts *inc_uts_namespaces(struct user_namespace *ns)
{
#include <linux/utsname.h>
#include <linux/sysctl.h>
#include <linux/wait.h>
+#include <linux/rwsem.h>
#ifdef CONFIG_PROC_SYSCTL
#include <linux/sysctl.h>
#include <linux/smpboot.h>
#include <linux/sched/rt.h>
+#include <uapi/linux/sched/types.h>
#include <linux/tick.h>
#include <linux/workqueue.h>
+#include <linux/sched/clock.h>
+#include <linux/sched/debug.h>
#include <asm/irq_regs.h>
#include <linux/kvm_para.h>
#include <linux/nmi.h>
#include <linux/module.h>
+#include <linux/sched/debug.h>
+
#include <asm/irq_regs.h>
#include <linux/perf_event.h>
#include <linux/kernel.h>
#include <linux/bug.h>
#include <linux/sched.h>
+#include <linux/rculist.h>
extern const struct bug_entry __start___bug_table[], __stop___bug_table[];
#include <linux/debugobjects.h>
#include <linux/interrupt.h>
#include <linux/sched.h>
+#include <linux/sched/task_stack.h>
#include <linux/seq_file.h>
#include <linux/debugfs.h>
#include <linux/slab.h>
struct pubkey_hdr *pkh;
down_read(&key->sem);
- ukp = user_key_payload(key);
+ ukp = user_key_payload_locked(key);
if (ukp->datalen < sizeof(*pkh))
goto err1;
* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
*/
+#include <linux/sched/task_stack.h>
#include <linux/scatterlist.h>
#include <linux/dma-mapping.h>
+#include <linux/sched/task.h>
#include <linux/stacktrace.h>
#include <linux/dma-debug.h>
#include <linux/spinlock.h>
#include <linux/kernel.h>
#include <linux/export.h>
#include <linux/sched.h>
+#include <linux/sched/debug.h>
#include <linux/smp.h>
#include <linux/atomic.h>
#include <asm/pgtable.h>
#ifdef CONFIG_HAVE_ARCH_HUGE_VMAP
+static int __read_mostly ioremap_p4d_capable;
static int __read_mostly ioremap_pud_capable;
static int __read_mostly ioremap_pmd_capable;
static int __read_mostly ioremap_huge_disabled;
}
}
+static inline int ioremap_p4d_enabled(void)
+{
+ return ioremap_p4d_capable;
+}
+
static inline int ioremap_pud_enabled(void)
{
return ioremap_pud_capable;
}
#else /* !CONFIG_HAVE_ARCH_HUGE_VMAP */
+static inline int ioremap_p4d_enabled(void) { return 0; }
static inline int ioremap_pud_enabled(void) { return 0; }
static inline int ioremap_pmd_enabled(void) { return 0; }
#endif /* CONFIG_HAVE_ARCH_HUGE_VMAP */
return 0;
}
-static inline int ioremap_pud_range(pgd_t *pgd, unsigned long addr,
+static inline int ioremap_pud_range(p4d_t *p4d, unsigned long addr,
unsigned long end, phys_addr_t phys_addr, pgprot_t prot)
{
pud_t *pud;
unsigned long next;
phys_addr -= addr;
- pud = pud_alloc(&init_mm, pgd, addr);
+ pud = pud_alloc(&init_mm, p4d, addr);
if (!pud)
return -ENOMEM;
do {
return 0;
}
+static inline int ioremap_p4d_range(pgd_t *pgd, unsigned long addr,
+ unsigned long end, phys_addr_t phys_addr, pgprot_t prot)
+{
+ p4d_t *p4d;
+ unsigned long next;
+
+ phys_addr -= addr;
+ p4d = p4d_alloc(&init_mm, pgd, addr);
+ if (!p4d)
+ return -ENOMEM;
+ do {
+ next = p4d_addr_end(addr, end);
+
+ if (ioremap_p4d_enabled() &&
+ ((next - addr) == P4D_SIZE) &&
+ IS_ALIGNED(phys_addr + addr, P4D_SIZE)) {
+ if (p4d_set_huge(p4d, phys_addr + addr, prot))
+ continue;
+ }
+
+ if (ioremap_pud_range(p4d, addr, next, phys_addr + addr, prot))
+ return -ENOMEM;
+ } while (p4d++, addr = next, addr != end);
+ return 0;
+}
+
int ioremap_page_range(unsigned long addr,
unsigned long end, phys_addr_t phys_addr, pgprot_t prot)
{
pgd = pgd_offset_k(addr);
do {
next = pgd_addr_end(addr, end);
- err = ioremap_pud_range(pgd, addr, next, phys_addr+addr, prot);
+ err = ioremap_p4d_range(pgd, addr, next, phys_addr+addr, prot);
if (err)
break;
} while (pgd++, addr = next, addr != end);
* as published by the Free Software Foundation; either version
* 2 of the Licence, or (at your option) any later version.
*/
-
-#include <linux/sched.h>
+#include <linux/sched/signal.h>
+#include <linux/sched/task.h>
+#include <linux/sched/mm.h>
/*
* Returns true if the task does not share ->mm with another thread/process.
#include <linux/kprobes.h>
#include <linux/nmi.h>
#include <linux/cpu.h>
+#include <linux/sched/debug.h>
#ifdef arch_trigger_cpumask_backtrace
/* For reliability, we're prepared to waste bits here. */
* General Public License for more details.
*/
+#include <linux/mm.h>
#include <linux/bitmap.h>
#include <linux/bitops.h>
#include <linux/bug.h>
#include <linux/init.h>
#include <linux/kernel.h>
#include <linux/percpu.h>
-#include <linux/sched.h>
+#include <linux/sched/signal.h>
#include <linux/string.h>
#include <linux/spinlock.h>
#include <linux/percpu_ida.h>
#ifdef CONFIG_DEBUG_PI_LIST
#include <linux/sched.h>
+#include <linux/sched/clock.h>
#include <linux/module.h>
#include <linux/init.h>
struct ida_bitmap *bitmap = kmalloc(sizeof(*bitmap), gfp);
if (!bitmap)
return 0;
- bitmap = this_cpu_cmpxchg(ida_bitmap, NULL, bitmap);
- kfree(bitmap);
+ if (this_cpu_cmpxchg(ida_bitmap, NULL, bitmap))
+ kfree(bitmap);
}
return 1;
val = old;
}
- WARN(new == UINT_MAX, "refcount_t: saturated; leaking memory.\n");
+ WARN_ONCE(new == UINT_MAX, "refcount_t: saturated; leaking memory.\n");
return true;
}
void refcount_add(unsigned int i, refcount_t *r)
{
- WARN(!refcount_add_not_zero(i, r), "refcount_t: addition on 0; use-after-free.\n");
+ WARN_ONCE(!refcount_add_not_zero(i, r), "refcount_t: addition on 0; use-after-free.\n");
}
EXPORT_SYMBOL_GPL(refcount_add);
val = old;
}
- WARN(new == UINT_MAX, "refcount_t: saturated; leaking memory.\n");
+ WARN_ONCE(new == UINT_MAX, "refcount_t: saturated; leaking memory.\n");
return true;
}
*/
void refcount_inc(refcount_t *r)
{
- WARN(!refcount_inc_not_zero(r), "refcount_t: increment on 0; use-after-free.\n");
+ WARN_ONCE(!refcount_inc_not_zero(r), "refcount_t: increment on 0; use-after-free.\n");
}
EXPORT_SYMBOL_GPL(refcount_inc);
new = val - i;
if (new > val) {
- WARN(new > val, "refcount_t: underflow; use-after-free.\n");
+ WARN_ONCE(new > val, "refcount_t: underflow; use-after-free.\n");
return false;
}
void refcount_dec(refcount_t *r)
{
- WARN(refcount_dec_and_test(r), "refcount_t: decrement hit 0; leaking memory.\n");
+ WARN_ONCE(refcount_dec_and_test(r), "refcount_t: decrement hit 0; leaking memory.\n");
}
EXPORT_SYMBOL_GPL(refcount_dec);
new = val - 1;
if (new > val) {
- WARN(new > val, "refcount_t: underflow; use-after-free.\n");
+ WARN_ONCE(new > val, "refcount_t: underflow; use-after-free.\n");
return true;
}
#include <linux/init.h>
#include <linux/log2.h>
#include <linux/sched.h>
+#include <linux/rculist.h>
#include <linux/slab.h>
#include <linux/vmalloc.h>
#include <linux/mm.h>
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
+#include <linux/sched.h>
#include <linux/random.h>
#include <linux/sbitmap.h>
#include <linux/seq_file.h>
* Kernel threads bound to a single CPU can safely use
* smp_processor_id():
*/
- if (cpumask_equal(tsk_cpus_allowed(current), cpumask_of(this_cpu)))
+ if (cpumask_equal(¤t->cpus_allowed, cpumask_of(this_cpu)))
goto out;
/*
#include <linux/ptrace.h>
#include <linux/sched.h>
+#include <linux/sched/task_stack.h>
#include <linux/export.h>
#include <asm/syscall.h>
static void cgwb_bdi_destroy(struct backing_dev_info *bdi)
{
struct radix_tree_iter iter;
- struct rb_node *rbn;
void **slot;
WARN_ON(test_bit(WB_registered, &bdi->wb.state));
spin_lock_irq(&cgwb_lock);
-
radix_tree_for_each_slot(slot, &bdi->cgwb_tree, &iter, 0)
cgwb_kill(*slot);
-
- while ((rbn = rb_first(&bdi->cgwb_congested_tree))) {
- struct bdi_writeback_congested *congested =
- rb_entry(rbn, struct bdi_writeback_congested, rb_node);
-
- rb_erase(rbn, &bdi->cgwb_congested_tree);
- congested->bdi = NULL; /* mark @congested unlinked */
- }
-
spin_unlock_irq(&cgwb_lock);
/*
- * All cgwb's and their congested states must be shutdown and
- * released before returning. Drain the usage counter to wait for
- * all cgwb's and cgwb_congested's ever created on @bdi.
+ * All cgwb's must be shutdown and released before returning. Drain
+ * the usage counter to wait for all cgwb's ever created on @bdi.
*/
atomic_dec(&bdi->usage_cnt);
wait_event(cgwb_release_wait, !atomic_read(&bdi->usage_cnt));
+ /*
+ * Grab back our reference so that we hold it when @bdi gets
+ * re-registered.
+ */
+ atomic_inc(&bdi->usage_cnt);
}
/**
spin_unlock_irq(&cgwb_lock);
}
+static void cgwb_bdi_exit(struct backing_dev_info *bdi)
+{
+ struct rb_node *rbn;
+
+ spin_lock_irq(&cgwb_lock);
+ while ((rbn = rb_first(&bdi->cgwb_congested_tree))) {
+ struct bdi_writeback_congested *congested =
+ rb_entry(rbn, struct bdi_writeback_congested, rb_node);
+
+ rb_erase(rbn, &bdi->cgwb_congested_tree);
+ congested->bdi = NULL; /* mark @congested unlinked */
+ }
+ spin_unlock_irq(&cgwb_lock);
+}
+
#else /* CONFIG_CGROUP_WRITEBACK */
static int cgwb_bdi_init(struct backing_dev_info *bdi)
return 0;
}
-static void cgwb_bdi_destroy(struct backing_dev_info *bdi)
+static void cgwb_bdi_destroy(struct backing_dev_info *bdi) { }
+
+static void cgwb_bdi_exit(struct backing_dev_info *bdi)
{
wb_congested_put(bdi->wb_congested);
}
MINOR(owner->devt));
if (rc)
return rc;
+ /* Leaking owner reference... */
+ WARN_ON(bdi->owner);
bdi->owner = owner;
get_device(owner);
return 0;
{
WARN_ON_ONCE(bdi->dev);
wb_exit(&bdi->wb);
+ cgwb_bdi_exit(bdi);
}
static void release_bdi(struct kref *ref)
#include <linux/migrate.h>
#include <linux/compaction.h>
#include <linux/mm_inline.h>
+#include <linux/sched/signal.h>
#include <linux/backing-dev.h>
#include <linux/sysctl.h>
#include <linux/sysfs.h>
#include <linux/compiler.h>
#include <linux/dax.h>
#include <linux/fs.h>
+#include <linux/sched/signal.h>
#include <linux/uaccess.h>
#include <linux/capability.h>
#include <linux/kernel_stat.h>
#include <linux/swap.h>
#include <linux/swapops.h>
-#include <linux/sched.h>
+#include <linux/sched/signal.h>
#include <linux/rwsem.h>
#include <linux/hugetlb.h>
unsigned int *page_mask)
{
pgd_t *pgd;
+ p4d_t *p4d;
pud_t *pud;
pmd_t *pmd;
spinlock_t *ptl;
pgd = pgd_offset(mm, address);
if (pgd_none(*pgd) || unlikely(pgd_bad(*pgd)))
return no_page_table(vma, flags);
-
- pud = pud_offset(pgd, address);
+ p4d = p4d_offset(pgd, address);
+ if (p4d_none(*p4d))
+ return no_page_table(vma, flags);
+ BUILD_BUG_ON(p4d_huge(*p4d));
+ if (unlikely(p4d_bad(*p4d)))
+ return no_page_table(vma, flags);
+ pud = pud_offset(p4d, address);
if (pud_none(*pud))
return no_page_table(vma, flags);
if (pud_huge(*pud) && vma->vm_flags & VM_HUGETLB) {
struct page **page)
{
pgd_t *pgd;
+ p4d_t *p4d;
pud_t *pud;
pmd_t *pmd;
pte_t *pte;
else
pgd = pgd_offset_gate(mm, address);
BUG_ON(pgd_none(*pgd));
- pud = pud_offset(pgd, address);
+ p4d = p4d_offset(pgd, address);
+ BUG_ON(p4d_none(*p4d));
+ pud = pud_offset(p4d, address);
BUG_ON(pud_none(*pud));
pmd = pmd_offset(pud, address);
if (pmd_none(*pmd))
return 1;
}
-static int gup_pud_range(pgd_t pgd, unsigned long addr, unsigned long end,
+static int gup_pud_range(p4d_t p4d, unsigned long addr, unsigned long end,
int write, struct page **pages, int *nr)
{
unsigned long next;
pud_t *pudp;
- pudp = pud_offset(&pgd, addr);
+ pudp = pud_offset(&p4d, addr);
do {
pud_t pud = READ_ONCE(*pudp);
return 1;
}
+static int gup_p4d_range(pgd_t pgd, unsigned long addr, unsigned long end,
+ int write, struct page **pages, int *nr)
+{
+ unsigned long next;
+ p4d_t *p4dp;
+
+ p4dp = p4d_offset(&pgd, addr);
+ do {
+ p4d_t p4d = READ_ONCE(*p4dp);
+
+ next = p4d_addr_end(addr, end);
+ if (p4d_none(p4d))
+ return 0;
+ BUILD_BUG_ON(p4d_huge(p4d));
+ if (unlikely(is_hugepd(__hugepd(p4d_val(p4d))))) {
+ if (!gup_huge_pd(__hugepd(p4d_val(p4d)), addr,
+ P4D_SHIFT, next, write, pages, nr))
+ return 0;
+ } else if (!gup_p4d_range(p4d, addr, next, write, pages, nr))
+ return 0;
+ } while (p4dp++, addr = next, addr != end);
+
+ return 1;
+}
+
/*
* Like get_user_pages_fast() except it's IRQ-safe in that it won't fall back to
* the regular GUP. It will only return non-negative values.
if (!gup_huge_pd(__hugepd(pgd_val(pgd)), addr,
PGDIR_SHIFT, next, write, pages, &nr))
break;
- } else if (!gup_pud_range(pgd, addr, next, write, pages, &nr))
+ } else if (!gup_p4d_range(pgd, addr, next, write, pages, &nr))
break;
} while (pgdp++, addr = next, addr != end);
local_irq_restore(flags);
#include <linux/mm.h>
#include <linux/sched.h>
+#include <linux/sched/coredump.h>
+#include <linux/sched/numa_balancing.h>
#include <linux/highmem.h>
#include <linux/hugetlb.h>
#include <linux/mmu_notifier.h>
VM_BUG_ON_VMA(vma->vm_end < haddr + HPAGE_PUD_SIZE, vma);
VM_BUG_ON(!pud_trans_huge(*pud) && !pud_devmap(*pud));
- count_vm_event(THP_SPLIT_PMD);
+ count_vm_event(THP_SPLIT_PUD);
pudp_huge_clear_flush_notify(vma, haddr, pud);
}
bool freeze, struct page *page)
{
pgd_t *pgd;
+ p4d_t *p4d;
pud_t *pud;
pmd_t *pmd;
if (!pgd_present(*pgd))
return;
- pud = pud_offset(pgd, address);
+ p4d = p4d_offset(pgd, address);
+ if (!p4d_present(*p4d))
+ return;
+
+ pud = pud_offset(p4d, address);
if (!pud_present(*pud))
return;
#include <linux/bootmem.h>
#include <linux/sysfs.h>
#include <linux/slab.h>
+#include <linux/sched/signal.h>
#include <linux/rmap.h>
#include <linux/swap.h>
#include <linux/swapops.h>
int huge_pmd_unshare(struct mm_struct *mm, unsigned long *addr, pte_t *ptep)
{
pgd_t *pgd = pgd_offset(mm, *addr);
- pud_t *pud = pud_offset(pgd, *addr);
+ p4d_t *p4d = p4d_offset(pgd, *addr);
+ pud_t *pud = pud_offset(p4d, *addr);
BUG_ON(page_count(virt_to_page(ptep)) == 0);
if (page_count(virt_to_page(ptep)) == 1)
unsigned long addr, unsigned long sz)
{
pgd_t *pgd;
+ p4d_t *p4d;
pud_t *pud;
pte_t *pte = NULL;
pgd = pgd_offset(mm, addr);
- pud = pud_alloc(mm, pgd, addr);
+ p4d = p4d_offset(pgd, addr);
+ pud = pud_alloc(mm, p4d, addr);
if (pud) {
if (sz == PUD_SIZE) {
pte = (pte_t *)pud;
pte_t *huge_pte_offset(struct mm_struct *mm, unsigned long addr)
{
pgd_t *pgd;
+ p4d_t *p4d;
pud_t *pud;
- pmd_t *pmd = NULL;
+ pmd_t *pmd;
pgd = pgd_offset(mm, addr);
- if (pgd_present(*pgd)) {
- pud = pud_offset(pgd, addr);
- if (pud_present(*pud)) {
- if (pud_huge(*pud))
- return (pte_t *)pud;
- pmd = pmd_offset(pud, addr);
- }
- }
+ if (!pgd_present(*pgd))
+ return NULL;
+ p4d = p4d_offset(pgd, addr);
+ if (!p4d_present(*p4d))
+ return NULL;
+ pud = pud_offset(p4d, addr);
+ if (!pud_present(*pud))
+ return NULL;
+ if (pud_huge(*pud))
+ return (pte_t *)pud;
+ pmd = pmd_offset(pud, addr);
return (pte_t *) pmd;
}
#include <linux/module.h>
#include <linux/printk.h>
#include <linux/sched.h>
+#include <linux/sched/task_stack.h>
#include <linux/slab.h>
#include <linux/stacktrace.h>
#include <linux/string.h>
#include "kasan.h"
#include "../slab.h"
+void kasan_enable_current(void)
+{
+ current->kasan_depth++;
+}
+
+void kasan_disable_current(void)
+{
+ current->kasan_depth--;
+}
+
/*
* Poisons the shadow memory for 'size' bytes starting from 'addr'.
* Memory addresses should be aligned to KASAN_SHADOW_SCALE_SIZE.
*/
unsigned char kasan_zero_page[PAGE_SIZE] __page_aligned_bss;
+#if CONFIG_PGTABLE_LEVELS > 4
+p4d_t kasan_zero_p4d[PTRS_PER_P4D] __page_aligned_bss;
+#endif
#if CONFIG_PGTABLE_LEVELS > 3
pud_t kasan_zero_pud[PTRS_PER_PUD] __page_aligned_bss;
#endif
} while (pmd++, addr = next, addr != end);
}
-static void __init zero_pud_populate(pgd_t *pgd, unsigned long addr,
+static void __init zero_pud_populate(p4d_t *p4d, unsigned long addr,
unsigned long end)
{
- pud_t *pud = pud_offset(pgd, addr);
+ pud_t *pud = pud_offset(p4d, addr);
unsigned long next;
do {
} while (pud++, addr = next, addr != end);
}
+static void __init zero_p4d_populate(pgd_t *pgd, unsigned long addr,
+ unsigned long end)
+{
+ p4d_t *p4d = p4d_offset(pgd, addr);
+ unsigned long next;
+
+ do {
+ next = p4d_addr_end(addr, end);
+
+ if (p4d_none(*p4d)) {
+ p4d_populate(&init_mm, p4d,
+ early_alloc(PAGE_SIZE, NUMA_NO_NODE));
+ }
+ zero_pud_populate(p4d, addr, next);
+ } while (p4d++, addr = next, addr != end);
+}
+
/**
* kasan_populate_zero_shadow - populate shadow memory region with
* kasan_zero_page
next = pgd_addr_end(addr, end);
if (IS_ALIGNED(addr, PGDIR_SIZE) && end - addr >= PGDIR_SIZE) {
+ p4d_t *p4d;
pud_t *pud;
pmd_t *pmd;
* 3,2 - level page tables where we don't have
* puds,pmds, so pgd_populate(), pud_populate()
* is noops.
+ *
+ * The ifndef is required to avoid build breakage.
+ *
+ * With 5level-fixup.h, pgd_populate() is not nop and
+ * we reference kasan_zero_p4d. It's not defined
+ * unless 5-level paging enabled.
+ *
+ * The ifndef can be dropped once all KASAN-enabled
+ * architectures will switch to pgtable-nop4d.h.
*/
- pgd_populate(&init_mm, pgd, lm_alias(kasan_zero_pud));
- pud = pud_offset(pgd, addr);
+#ifndef __ARCH_HAS_5LEVEL_HACK
+ pgd_populate(&init_mm, pgd, lm_alias(kasan_zero_p4d));
+#endif
+ p4d = p4d_offset(pgd, addr);
+ p4d_populate(&init_mm, p4d, lm_alias(kasan_zero_pud));
+ pud = pud_offset(p4d, addr);
pud_populate(&init_mm, pud, lm_alias(kasan_zero_pmd));
pmd = pmd_offset(pud, addr);
pmd_populate_kernel(&init_mm, pmd, lm_alias(kasan_zero_pte));
pgd_populate(&init_mm, pgd,
early_alloc(PAGE_SIZE, NUMA_NO_NODE));
}
- zero_pud_populate(pgd, addr, next);
+ zero_p4d_populate(pgd, addr, next);
} while (pgd++, addr = next, addr != end);
}
#include <linux/printk.h>
#include <linux/shrinker.h>
#include <linux/slab.h>
+#include <linux/srcu.h>
#include <linux/string.h>
#include <linux/types.h>
/* Total size of all objects in global_quarantine across all batches. */
static unsigned long quarantine_size;
static DEFINE_SPINLOCK(quarantine_lock);
+DEFINE_STATIC_SRCU(remove_cache_srcu);
/* Maximum size of the global queue. */
static unsigned long quarantine_max_size;
struct qlist_head *q;
struct qlist_head temp = QLIST_INIT;
+ /*
+ * Note: irq must be disabled until after we move the batch to the
+ * global quarantine. Otherwise quarantine_remove_cache() can miss
+ * some objects belonging to the cache if they are in our local temp
+ * list. quarantine_remove_cache() executes on_each_cpu() at the
+ * beginning which ensures that it either sees the objects in per-cpu
+ * lists or in the global quarantine.
+ */
local_irq_save(flags);
q = this_cpu_ptr(&cpu_quarantine);
qlist_put(q, &info->quarantine_link, cache->size);
- if (unlikely(q->bytes > QUARANTINE_PERCPU_SIZE))
+ if (unlikely(q->bytes > QUARANTINE_PERCPU_SIZE)) {
qlist_move_all(q, &temp);
- local_irq_restore(flags);
-
- if (unlikely(!qlist_empty(&temp))) {
- spin_lock_irqsave(&quarantine_lock, flags);
+ spin_lock(&quarantine_lock);
WRITE_ONCE(quarantine_size, quarantine_size + temp.bytes);
qlist_move_all(&temp, &global_quarantine[quarantine_tail]);
if (global_quarantine[quarantine_tail].bytes >=
if (new_tail != quarantine_head)
quarantine_tail = new_tail;
}
- spin_unlock_irqrestore(&quarantine_lock, flags);
+ spin_unlock(&quarantine_lock);
}
+
+ local_irq_restore(flags);
}
void quarantine_reduce(void)
{
size_t total_size, new_quarantine_size, percpu_quarantines;
unsigned long flags;
+ int srcu_idx;
struct qlist_head to_free = QLIST_INIT;
if (likely(READ_ONCE(quarantine_size) <=
READ_ONCE(quarantine_max_size)))
return;
+ /*
+ * srcu critical section ensures that quarantine_remove_cache()
+ * will not miss objects belonging to the cache while they are in our
+ * local to_free list. srcu is chosen because (1) it gives us private
+ * grace period domain that does not interfere with anything else,
+ * and (2) it allows synchronize_srcu() to return without waiting
+ * if there are no pending read critical sections (which is the
+ * expected case).
+ */
+ srcu_idx = srcu_read_lock(&remove_cache_srcu);
spin_lock_irqsave(&quarantine_lock, flags);
/*
spin_unlock_irqrestore(&quarantine_lock, flags);
qlist_free_all(&to_free, NULL);
+ srcu_read_unlock(&remove_cache_srcu, srcu_idx);
}
static void qlist_move_cache(struct qlist_head *from,
unsigned long flags, i;
struct qlist_head to_free = QLIST_INIT;
+ /*
+ * Must be careful to not miss any objects that are being moved from
+ * per-cpu list to the global quarantine in quarantine_put(),
+ * nor objects being freed in quarantine_reduce(). on_each_cpu()
+ * achieves the first goal, while synchronize_srcu() achieves the
+ * second.
+ */
on_each_cpu(per_cpu_remove_cache, cache, 1);
spin_lock_irqsave(&quarantine_lock, flags);
- for (i = 0; i < QUARANTINE_BATCHES; i++)
+ for (i = 0; i < QUARANTINE_BATCHES; i++) {
+ if (qlist_empty(&global_quarantine[i]))
+ continue;
qlist_move_cache(&global_quarantine[i], &to_free, cache);
+ /* Scanning whole quarantine can take a while. */
+ spin_unlock_irqrestore(&quarantine_lock, flags);
+ cond_resched();
+ spin_lock_irqsave(&quarantine_lock, flags);
+ }
spin_unlock_irqrestore(&quarantine_lock, flags);
qlist_free_all(&to_free, cache);
+
+ synchronize_srcu(&remove_cache_srcu);
}
#include <linux/mm.h>
#include <linux/sched.h>
+#include <linux/sched/mm.h>
+#include <linux/sched/coredump.h>
#include <linux/mmu_notifier.h>
#include <linux/rmap.h>
#include <linux/swap.h>
#include <linux/init.h>
#include <linux/kernel.h>
#include <linux/list.h>
-#include <linux/sched.h>
+#include <linux/sched/signal.h>
+#include <linux/sched/task.h>
+#include <linux/sched/task_stack.h>
#include <linux/jiffies.h>
#include <linux/delay.h>
#include <linux/export.h>
#include <linux/fs.h>
#include <linux/mman.h>
#include <linux/sched.h>
+#include <linux/sched/mm.h>
+#include <linux/sched/coredump.h>
#include <linux/rwsem.h>
#include <linux/pagemap.h>
#include <linux/rmap.h>
if (!can_madv_dontneed_vma(vma))
return -EINVAL;
- userfaultfd_remove(vma, prev, start, end);
+ if (!userfaultfd_remove(vma, start, end)) {
+ *prev = NULL; /* mmap_sem has been dropped, prev is stale */
+
+ down_read(¤t->mm->mmap_sem);
+ vma = find_vma(current->mm, start);
+ if (!vma)
+ return -ENOMEM;
+ if (start < vma->vm_start) {
+ /*
+ * This "vma" under revalidation is the one
+ * with the lowest vma->vm_start where start
+ * is also < vma->vm_end. If start <
+ * vma->vm_start it means an hole materialized
+ * in the user address space within the
+ * virtual range passed to MADV_DONTNEED.
+ */
+ return -ENOMEM;
+ }
+ if (!can_madv_dontneed_vma(vma))
+ return -EINVAL;
+ if (end > vma->vm_end) {
+ /*
+ * Don't fail if end > vma->vm_end. If the old
+ * vma was splitted while the mmap_sem was
+ * released the effect of the concurrent
+ * operation may not cause MADV_DONTNEED to
+ * have an undefined result. There may be an
+ * adjacent next vma that we'll walk
+ * next. userfaultfd_remove() will generate an
+ * UFFD_EVENT_REMOVE repetition on the
+ * end-vma->vm_end range, but the manager can
+ * handle a repetition fine.
+ */
+ end = vma->vm_end;
+ }
+ VM_WARN_ON(start >= end);
+ }
zap_page_range(vma, start, end - start);
return 0;
}
* mmap_sem.
*/
get_file(f);
- userfaultfd_remove(vma, prev, start, end);
- up_read(¤t->mm->mmap_sem);
+ if (userfaultfd_remove(vma, start, end)) {
+ /* mmap_sem was not released by userfaultfd_remove() */
+ up_read(¤t->mm->mmap_sem);
+ }
error = vfs_fallocate(f,
FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE,
offset, end - start);
}
} while (left < right);
- return min(PHYS_PFN(type->regions[right].base), max_pfn);
+ if (right == type->cnt)
+ return max_pfn;
+ else
+ return min(PHYS_PFN(type->regions[right].base), max_pfn);
}
/**
#include <linux/memcontrol.h>
#include <linux/cgroup.h>
#include <linux/mm.h>
+#include <linux/sched/mm.h>
#include <linux/shmem_fs.h>
#include <linux/hugetlb.h>
#include <linux/pagemap.h>
struct mem_cgroup_tree_per_node *mctz;
mctz = soft_limit_tree_from_page(page);
+ if (!mctz)
+ return;
/*
* Necessary to update all ancestors when hierarchy is used.
* because their event counter is not touched.
for_each_node(nid) {
mz = mem_cgroup_nodeinfo(memcg, nid);
mctz = soft_limit_tree_node(nid);
- mem_cgroup_remove_exceeded(mz, mctz);
+ if (mctz)
+ mem_cgroup_remove_exceeded(mz, mctz);
}
}
* is empty. Do it lockless to prevent lock bouncing. Races
* are acceptable as soft limit is best effort anyway.
*/
- if (RB_EMPTY_ROOT(&mctz->rb_root))
+ if (!mctz || RB_EMPTY_ROOT(&mctz->rb_root))
return 0;
/*
kfree(memcg->nodeinfo[node]);
}
-static void mem_cgroup_free(struct mem_cgroup *memcg)
+static void __mem_cgroup_free(struct mem_cgroup *memcg)
{
int node;
- memcg_wb_domain_exit(memcg);
for_each_node(node)
free_mem_cgroup_per_node_info(memcg, node);
free_percpu(memcg->stat);
kfree(memcg);
}
+static void mem_cgroup_free(struct mem_cgroup *memcg)
+{
+ memcg_wb_domain_exit(memcg);
+ __mem_cgroup_free(memcg);
+}
+
static struct mem_cgroup *mem_cgroup_alloc(void)
{
struct mem_cgroup *memcg;
fail:
if (memcg->id.id > 0)
idr_remove(&mem_cgroup_idr, memcg->id.id);
- mem_cgroup_free(memcg);
+ __mem_cgroup_free(memcg);
return NULL;
}
#include <linux/mm.h>
#include <linux/page-flags.h>
#include <linux/kernel-page-flags.h>
-#include <linux/sched.h>
+#include <linux/sched/signal.h>
+#include <linux/sched/task.h>
#include <linux/ksm.h>
#include <linux/rmap.h>
#include <linux/export.h>
#include <linux/kernel_stat.h>
#include <linux/mm.h>
+#include <linux/sched/mm.h>
+#include <linux/sched/coredump.h>
+#include <linux/sched/numa_balancing.h>
+#include <linux/sched/task.h>
#include <linux/hugetlb.h>
#include <linux/mman.h>
#include <linux/swap.h>
mm_dec_nr_pmds(tlb->mm);
}
-static inline void free_pud_range(struct mmu_gather *tlb, pgd_t *pgd,
+static inline void free_pud_range(struct mmu_gather *tlb, p4d_t *p4d,
unsigned long addr, unsigned long end,
unsigned long floor, unsigned long ceiling)
{
unsigned long start;
start = addr;
- pud = pud_offset(pgd, addr);
+ pud = pud_offset(p4d, addr);
do {
next = pud_addr_end(addr, end);
if (pud_none_or_clear_bad(pud))
free_pmd_range(tlb, pud, addr, next, floor, ceiling);
} while (pud++, addr = next, addr != end);
+ start &= P4D_MASK;
+ if (start < floor)
+ return;
+ if (ceiling) {
+ ceiling &= P4D_MASK;
+ if (!ceiling)
+ return;
+ }
+ if (end - 1 > ceiling - 1)
+ return;
+
+ pud = pud_offset(p4d, start);
+ p4d_clear(p4d);
+ pud_free_tlb(tlb, pud, start);
+}
+
+static inline void free_p4d_range(struct mmu_gather *tlb, pgd_t *pgd,
+ unsigned long addr, unsigned long end,
+ unsigned long floor, unsigned long ceiling)
+{
+ p4d_t *p4d;
+ unsigned long next;
+ unsigned long start;
+
+ start = addr;
+ p4d = p4d_offset(pgd, addr);
+ do {
+ next = p4d_addr_end(addr, end);
+ if (p4d_none_or_clear_bad(p4d))
+ continue;
+ free_pud_range(tlb, p4d, addr, next, floor, ceiling);
+ } while (p4d++, addr = next, addr != end);
+
start &= PGDIR_MASK;
if (start < floor)
return;
if (end - 1 > ceiling - 1)
return;
- pud = pud_offset(pgd, start);
+ p4d = p4d_offset(pgd, start);
pgd_clear(pgd);
- pud_free_tlb(tlb, pud, start);
+ p4d_free_tlb(tlb, p4d, start);
}
/*
next = pgd_addr_end(addr, end);
if (pgd_none_or_clear_bad(pgd))
continue;
- free_pud_range(tlb, pgd, addr, next, floor, ceiling);
+ free_p4d_range(tlb, pgd, addr, next, floor, ceiling);
} while (pgd++, addr = next, addr != end);
}
pte_t pte, struct page *page)
{
pgd_t *pgd = pgd_offset(vma->vm_mm, addr);
- pud_t *pud = pud_offset(pgd, addr);
+ p4d_t *p4d = p4d_offset(pgd, addr);
+ pud_t *pud = pud_offset(p4d, addr);
pmd_t *pmd = pmd_offset(pud, addr);
struct address_space *mapping;
pgoff_t index;
}
static inline int copy_pud_range(struct mm_struct *dst_mm, struct mm_struct *src_mm,
- pgd_t *dst_pgd, pgd_t *src_pgd, struct vm_area_struct *vma,
+ p4d_t *dst_p4d, p4d_t *src_p4d, struct vm_area_struct *vma,
unsigned long addr, unsigned long end)
{
pud_t *src_pud, *dst_pud;
unsigned long next;
- dst_pud = pud_alloc(dst_mm, dst_pgd, addr);
+ dst_pud = pud_alloc(dst_mm, dst_p4d, addr);
if (!dst_pud)
return -ENOMEM;
- src_pud = pud_offset(src_pgd, addr);
+ src_pud = pud_offset(src_p4d, addr);
do {
next = pud_addr_end(addr, end);
if (pud_trans_huge(*src_pud) || pud_devmap(*src_pud)) {
return 0;
}
+static inline int copy_p4d_range(struct mm_struct *dst_mm, struct mm_struct *src_mm,
+ pgd_t *dst_pgd, pgd_t *src_pgd, struct vm_area_struct *vma,
+ unsigned long addr, unsigned long end)
+{
+ p4d_t *src_p4d, *dst_p4d;
+ unsigned long next;
+
+ dst_p4d = p4d_alloc(dst_mm, dst_pgd, addr);
+ if (!dst_p4d)
+ return -ENOMEM;
+ src_p4d = p4d_offset(src_pgd, addr);
+ do {
+ next = p4d_addr_end(addr, end);
+ if (p4d_none_or_clear_bad(src_p4d))
+ continue;
+ if (copy_pud_range(dst_mm, src_mm, dst_p4d, src_p4d,
+ vma, addr, next))
+ return -ENOMEM;
+ } while (dst_p4d++, src_p4d++, addr = next, addr != end);
+ return 0;
+}
+
int copy_page_range(struct mm_struct *dst_mm, struct mm_struct *src_mm,
struct vm_area_struct *vma)
{
next = pgd_addr_end(addr, end);
if (pgd_none_or_clear_bad(src_pgd))
continue;
- if (unlikely(copy_pud_range(dst_mm, src_mm, dst_pgd, src_pgd,
+ if (unlikely(copy_p4d_range(dst_mm, src_mm, dst_pgd, src_pgd,
vma, addr, next))) {
ret = -ENOMEM;
break;
}
static inline unsigned long zap_pud_range(struct mmu_gather *tlb,
- struct vm_area_struct *vma, pgd_t *pgd,
+ struct vm_area_struct *vma, p4d_t *p4d,
unsigned long addr, unsigned long end,
struct zap_details *details)
{
pud_t *pud;
unsigned long next;
- pud = pud_offset(pgd, addr);
+ pud = pud_offset(p4d, addr);
do {
next = pud_addr_end(addr, end);
if (pud_trans_huge(*pud) || pud_devmap(*pud)) {
return addr;
}
+static inline unsigned long zap_p4d_range(struct mmu_gather *tlb,
+ struct vm_area_struct *vma, pgd_t *pgd,
+ unsigned long addr, unsigned long end,
+ struct zap_details *details)
+{
+ p4d_t *p4d;
+ unsigned long next;
+
+ p4d = p4d_offset(pgd, addr);
+ do {
+ next = p4d_addr_end(addr, end);
+ if (p4d_none_or_clear_bad(p4d))
+ continue;
+ next = zap_pud_range(tlb, vma, p4d, addr, next, details);
+ } while (p4d++, addr = next, addr != end);
+
+ return addr;
+}
+
void unmap_page_range(struct mmu_gather *tlb,
struct vm_area_struct *vma,
unsigned long addr, unsigned long end,
next = pgd_addr_end(addr, end);
if (pgd_none_or_clear_bad(pgd))
continue;
- next = zap_pud_range(tlb, vma, pgd, addr, next, details);
+ next = zap_p4d_range(tlb, vma, pgd, addr, next, details);
} while (pgd++, addr = next, addr != end);
tlb_end_vma(tlb, vma);
}
pte_t *__get_locked_pte(struct mm_struct *mm, unsigned long addr,
spinlock_t **ptl)
{
- pgd_t *pgd = pgd_offset(mm, addr);
- pud_t *pud = pud_alloc(mm, pgd, addr);
- if (pud) {
- pmd_t *pmd = pmd_alloc(mm, pud, addr);
- if (pmd) {
- VM_BUG_ON(pmd_trans_huge(*pmd));
- return pte_alloc_map_lock(mm, pmd, addr, ptl);
- }
- }
- return NULL;
+ pgd_t *pgd;
+ p4d_t *p4d;
+ pud_t *pud;
+ pmd_t *pmd;
+
+ pgd = pgd_offset(mm, addr);
+ p4d = p4d_alloc(mm, pgd, addr);
+ if (!p4d)
+ return NULL;
+ pud = pud_alloc(mm, p4d, addr);
+ if (!pud)
+ return NULL;
+ pmd = pmd_alloc(mm, pud, addr);
+ if (!pmd)
+ return NULL;
+
+ VM_BUG_ON(pmd_trans_huge(*pmd));
+ return pte_alloc_map_lock(mm, pmd, addr, ptl);
}
/*
return 0;
}
-static inline int remap_pud_range(struct mm_struct *mm, pgd_t *pgd,
+static inline int remap_pud_range(struct mm_struct *mm, p4d_t *p4d,
unsigned long addr, unsigned long end,
unsigned long pfn, pgprot_t prot)
{
unsigned long next;
pfn -= addr >> PAGE_SHIFT;
- pud = pud_alloc(mm, pgd, addr);
+ pud = pud_alloc(mm, p4d, addr);
if (!pud)
return -ENOMEM;
do {
return 0;
}
+static inline int remap_p4d_range(struct mm_struct *mm, pgd_t *pgd,
+ unsigned long addr, unsigned long end,
+ unsigned long pfn, pgprot_t prot)
+{
+ p4d_t *p4d;
+ unsigned long next;
+
+ pfn -= addr >> PAGE_SHIFT;
+ p4d = p4d_alloc(mm, pgd, addr);
+ if (!p4d)
+ return -ENOMEM;
+ do {
+ next = p4d_addr_end(addr, end);
+ if (remap_pud_range(mm, p4d, addr, next,
+ pfn + (addr >> PAGE_SHIFT), prot))
+ return -ENOMEM;
+ } while (p4d++, addr = next, addr != end);
+ return 0;
+}
+
/**
* remap_pfn_range - remap kernel memory to userspace
* @vma: user vma to map to
flush_cache_range(vma, addr, end);
do {
next = pgd_addr_end(addr, end);
- err = remap_pud_range(mm, pgd, addr, next,
+ err = remap_p4d_range(mm, pgd, addr, next,
pfn + (addr >> PAGE_SHIFT), prot);
if (err)
break;
return err;
}
-static int apply_to_pud_range(struct mm_struct *mm, pgd_t *pgd,
+static int apply_to_pud_range(struct mm_struct *mm, p4d_t *p4d,
unsigned long addr, unsigned long end,
pte_fn_t fn, void *data)
{
unsigned long next;
int err;
- pud = pud_alloc(mm, pgd, addr);
+ pud = pud_alloc(mm, p4d, addr);
if (!pud)
return -ENOMEM;
do {
return err;
}
+static int apply_to_p4d_range(struct mm_struct *mm, pgd_t *pgd,
+ unsigned long addr, unsigned long end,
+ pte_fn_t fn, void *data)
+{
+ p4d_t *p4d;
+ unsigned long next;
+ int err;
+
+ p4d = p4d_alloc(mm, pgd, addr);
+ if (!p4d)
+ return -ENOMEM;
+ do {
+ next = p4d_addr_end(addr, end);
+ err = apply_to_pud_range(mm, p4d, addr, next, fn, data);
+ if (err)
+ break;
+ } while (p4d++, addr = next, addr != end);
+ return err;
+}
+
/*
* Scan a region of virtual memory, filling in page tables as necessary
* and calling a provided function on each leaf page table.
pgd = pgd_offset(mm, addr);
do {
next = pgd_addr_end(addr, end);
- err = apply_to_pud_range(mm, pgd, addr, next, fn, data);
+ err = apply_to_p4d_range(mm, pgd, addr, next, fn, data);
if (err)
break;
} while (pgd++, addr = next, addr != end);
};
struct mm_struct *mm = vma->vm_mm;
pgd_t *pgd;
+ p4d_t *p4d;
int ret;
pgd = pgd_offset(mm, address);
+ p4d = p4d_alloc(mm, pgd, address);
+ if (!p4d)
+ return VM_FAULT_OOM;
- vmf.pud = pud_alloc(mm, pgd, address);
+ vmf.pud = pud_alloc(mm, p4d, address);
if (!vmf.pud)
return VM_FAULT_OOM;
if (pud_none(*vmf.pud) && transparent_hugepage_enabled(vma)) {
}
EXPORT_SYMBOL_GPL(handle_mm_fault);
+#ifndef __PAGETABLE_P4D_FOLDED
+/*
+ * Allocate p4d page table.
+ * We've already handled the fast-path in-line.
+ */
+int __p4d_alloc(struct mm_struct *mm, pgd_t *pgd, unsigned long address)
+{
+ p4d_t *new = p4d_alloc_one(mm, address);
+ if (!new)
+ return -ENOMEM;
+
+ smp_wmb(); /* See comment in __pte_alloc */
+
+ spin_lock(&mm->page_table_lock);
+ if (pgd_present(*pgd)) /* Another has populated it */
+ p4d_free(mm, new);
+ else
+ pgd_populate(mm, pgd, new);
+ spin_unlock(&mm->page_table_lock);
+ return 0;
+}
+#endif /* __PAGETABLE_P4D_FOLDED */
+
#ifndef __PAGETABLE_PUD_FOLDED
/*
* Allocate page upper directory.
* We've already handled the fast-path in-line.
*/
-int __pud_alloc(struct mm_struct *mm, pgd_t *pgd, unsigned long address)
+int __pud_alloc(struct mm_struct *mm, p4d_t *p4d, unsigned long address)
{
pud_t *new = pud_alloc_one(mm, address);
if (!new)
smp_wmb(); /* See comment in __pte_alloc */
spin_lock(&mm->page_table_lock);
- if (pgd_present(*pgd)) /* Another has populated it */
+#ifndef __ARCH_HAS_5LEVEL_HACK
+ if (p4d_present(*p4d)) /* Another has populated it */
pud_free(mm, new);
else
- pgd_populate(mm, pgd, new);
+ p4d_populate(mm, p4d, new);
+#else
+ if (pgd_present(*p4d)) /* Another has populated it */
+ pud_free(mm, new);
+ else
+ pgd_populate(mm, p4d, new);
+#endif /* __ARCH_HAS_5LEVEL_HACK */
spin_unlock(&mm->page_table_lock);
return 0;
}
pte_t **ptepp, pmd_t **pmdpp, spinlock_t **ptlp)
{
pgd_t *pgd;
+ p4d_t *p4d;
pud_t *pud;
pmd_t *pmd;
pte_t *ptep;
if (pgd_none(*pgd) || unlikely(pgd_bad(*pgd)))
goto out;
- pud = pud_offset(pgd, address);
+ p4d = p4d_offset(pgd, address);
+ if (p4d_none(*p4d) || unlikely(p4d_bad(*p4d)))
+ goto out;
+
+ pud = pud_offset(p4d, address);
if (pud_none(*pud) || unlikely(pud_bad(*pud)))
goto out;
#include <linux/stddef.h>
#include <linux/mm.h>
+#include <linux/sched/signal.h>
#include <linux/swap.h>
#include <linux/interrupt.h>
#include <linux/pagemap.h>
#include <linux/hugetlb.h>
#include <linux/kernel.h>
#include <linux/sched.h>
+#include <linux/sched/mm.h>
+#include <linux/sched/numa_balancing.h>
+#include <linux/sched/task.h>
#include <linux/nodemask.h>
#include <linux/cpuset.h>
#include <linux/slab.h>
#include <linux/mmu_notifier.h>
#include <linux/page_idle.h>
#include <linux/page_owner.h>
+#include <linux/sched/mm.h>
#include <asm/tlbflush.h>
#include <linux/capability.h>
#include <linux/mman.h>
#include <linux/mm.h>
+#include <linux/sched/user.h>
#include <linux/swap.h>
#include <linux/swapops.h>
#include <linux/pagemap.h>
pte = get_locked_pte(vma->vm_mm, start, &ptl);
/* Make sure we do not cross the page table boundary */
end = pgd_addr_end(start, end);
+ end = p4d_addr_end(start, end);
end = pud_addr_end(start, end);
end = pmd_addr_end(start, end);
while (start < end) {
struct page *page;
- unsigned int page_mask;
+ unsigned int page_mask = 0;
unsigned long page_increm;
struct pagevec pvec;
struct zone *zone;
* suits munlock very well (and if somehow an abnormal page
* has sneaked into the range, we won't oops here: great).
*/
- page = follow_page_mask(vma, start, FOLL_GET | FOLL_DUMP,
- &page_mask);
+ page = follow_page(vma, start, FOLL_GET | FOLL_DUMP);
if (page && !IS_ERR(page)) {
if (PageTransTail(page)) {
/*
* Any THP page found by follow_page_mask() may
* have gotten split before reaching
- * munlock_vma_page(), so we need to recompute
- * the page_mask here.
+ * munlock_vma_page(), so we need to compute
+ * the page_mask here instead.
*/
page_mask = munlock_vma_page(page);
unlock_page(page);
* new file must not have been exposed to user-space, yet.
*/
vma->vm_file = get_file(file);
- error = file->f_op->mmap(file, vma);
+ error = call_mmap(file, vma);
if (error)
goto unmap_and_free_vma;
#include <linux/mm.h>
#include <linux/sched.h>
+#include <linux/sched/mm.h>
+#include <linux/sched/task.h>
#include <linux/mmu_context.h>
#include <linux/export.h>
#include <linux/srcu.h>
#include <linux/rcupdate.h>
#include <linux/sched.h>
+#include <linux/sched/mm.h>
#include <linux/slab.h>
/* global SRCU for all MMs */
}
static inline unsigned long change_pud_range(struct vm_area_struct *vma,
- pgd_t *pgd, unsigned long addr, unsigned long end,
+ p4d_t *p4d, unsigned long addr, unsigned long end,
pgprot_t newprot, int dirty_accountable, int prot_numa)
{
pud_t *pud;
unsigned long next;
unsigned long pages = 0;
- pud = pud_offset(pgd, addr);
+ pud = pud_offset(p4d, addr);
do {
next = pud_addr_end(addr, end);
if (pud_none_or_clear_bad(pud))
return pages;
}
+static inline unsigned long change_p4d_range(struct vm_area_struct *vma,
+ pgd_t *pgd, unsigned long addr, unsigned long end,
+ pgprot_t newprot, int dirty_accountable, int prot_numa)
+{
+ p4d_t *p4d;
+ unsigned long next;
+ unsigned long pages = 0;
+
+ p4d = p4d_offset(pgd, addr);
+ do {
+ next = p4d_addr_end(addr, end);
+ if (p4d_none_or_clear_bad(p4d))
+ continue;
+ pages += change_pud_range(vma, p4d, addr, next, newprot,
+ dirty_accountable, prot_numa);
+ } while (p4d++, addr = next, addr != end);
+
+ return pages;
+}
+
static unsigned long change_protection_range(struct vm_area_struct *vma,
unsigned long addr, unsigned long end, pgprot_t newprot,
int dirty_accountable, int prot_numa)
next = pgd_addr_end(addr, end);
if (pgd_none_or_clear_bad(pgd))
continue;
- pages += change_pud_range(vma, pgd, addr, next, newprot,
+ pages += change_p4d_range(vma, pgd, addr, next, newprot,
dirty_accountable, prot_numa);
} while (pgd++, addr = next, addr != end);
static pmd_t *get_old_pmd(struct mm_struct *mm, unsigned long addr)
{
pgd_t *pgd;
+ p4d_t *p4d;
pud_t *pud;
pmd_t *pmd;
if (pgd_none_or_clear_bad(pgd))
return NULL;
- pud = pud_offset(pgd, addr);
+ p4d = p4d_offset(pgd, addr);
+ if (p4d_none_or_clear_bad(p4d))
+ return NULL;
+
+ pud = pud_offset(p4d, addr);
if (pud_none_or_clear_bad(pud))
return NULL;
unsigned long addr)
{
pgd_t *pgd;
+ p4d_t *p4d;
pud_t *pud;
pmd_t *pmd;
pgd = pgd_offset(mm, addr);
- pud = pud_alloc(mm, pgd, addr);
+ p4d = p4d_alloc(mm, pgd, addr);
+ if (!p4d)
+ return NULL;
+ pud = pud_alloc(mm, p4d, addr);
if (!pud)
return NULL;
#include <linux/export.h>
#include <linux/mm.h>
+#include <linux/sched/mm.h>
#include <linux/vmacache.h>
#include <linux/mman.h>
#include <linux/swap.h>
mm->map_count--;
for (i = 0; i < VMACACHE_SIZE; i++) {
/* if the vma is cached, invalidate the entire cache */
- if (curr->vmacache[i] == vma) {
+ if (curr->vmacache.vmas[i] == vma) {
vmacache_invalidate(mm);
break;
}
{
int ret;
- ret = vma->vm_file->f_op->mmap(vma->vm_file, vma);
+ ret = call_mmap(vma->vm_file, vma);
if (ret == 0) {
vma->vm_region->vm_top = vma->vm_region->vm_end;
return 0;
* - VM_MAYSHARE will be set if it may attempt to share
*/
if (capabilities & NOMMU_MAP_DIRECT) {
- ret = vma->vm_file->f_op->mmap(vma->vm_file, vma);
+ ret = call_mmap(vma->vm_file, vma);
if (ret == 0) {
/* shouldn't return success if we're not sharing */
BUG_ON(!(vma->vm_flags & VM_MAYSHARE));
#include <linux/err.h>
#include <linux/gfp.h>
#include <linux/sched.h>
+#include <linux/sched/mm.h>
+#include <linux/sched/coredump.h>
+#include <linux/sched/task.h>
#include <linux/swap.h>
#include <linux/timex.h>
#include <linux/jiffies.h>
#include <linux/pagevec.h>
#include <linux/timer.h>
#include <linux/sched/rt.h>
+#include <linux/sched/signal.h>
#include <linux/mm_inline.h>
#include <trace/events/writeback.h>
#include <linux/migrate.h>
#include <linux/hugetlb.h>
#include <linux/sched/rt.h>
+#include <linux/sched/mm.h>
#include <linux/page_owner.h>
#include <linux/kthread.h>
#include <linux/memcontrol.h>
higher_page = page + (combined_pfn - pfn);
buddy_pfn = __find_buddy_pfn(combined_pfn, order + 1);
higher_buddy = higher_page + (buddy_pfn - combined_pfn);
- if (page_is_buddy(higher_page, higher_buddy, order + 1)) {
+ if (pfn_valid_within(buddy_pfn) &&
+ page_is_buddy(higher_page, higher_buddy, order + 1)) {
list_add_tail(&page->lru,
&zone->free_area[order].free_list[migratetype]);
goto out;
struct mm_struct *mm = pvmw->vma->vm_mm;
struct page *page = pvmw->page;
pgd_t *pgd;
+ p4d_t *p4d;
pud_t *pud;
/* The only possible pmd mapping has been handled on last iteration */
pgd = pgd_offset(mm, pvmw->address);
if (!pgd_present(*pgd))
return false;
- pud = pud_offset(pgd, pvmw->address);
+ p4d = p4d_offset(pgd, pvmw->address);
+ if (!p4d_present(*p4d))
+ return false;
+ pud = pud_offset(p4d, pvmw->address);
if (!pud_present(*pud))
return false;
pvmw->pmd = pmd_offset(pud, pvmw->address);
return err;
}
-static int walk_pud_range(pgd_t *pgd, unsigned long addr, unsigned long end,
+static int walk_pud_range(p4d_t *p4d, unsigned long addr, unsigned long end,
struct mm_walk *walk)
{
pud_t *pud;
unsigned long next;
int err = 0;
- pud = pud_offset(pgd, addr);
+ pud = pud_offset(p4d, addr);
do {
again:
next = pud_addr_end(addr, end);
return err;
}
+static int walk_p4d_range(pgd_t *pgd, unsigned long addr, unsigned long end,
+ struct mm_walk *walk)
+{
+ p4d_t *p4d;
+ unsigned long next;
+ int err = 0;
+
+ p4d = p4d_offset(pgd, addr);
+ do {
+ next = p4d_addr_end(addr, end);
+ if (p4d_none_or_clear_bad(p4d)) {
+ if (walk->pte_hole)
+ err = walk->pte_hole(addr, next, walk);
+ if (err)
+ break;
+ continue;
+ }
+ if (walk->pmd_entry || walk->pte_entry)
+ err = walk_pud_range(p4d, addr, next, walk);
+ if (err)
+ break;
+ } while (p4d++, addr = next, addr != end);
+
+ return err;
+}
+
static int walk_pgd_range(unsigned long addr, unsigned long end,
struct mm_walk *walk)
{
continue;
}
if (walk->pmd_entry || walk->pte_entry)
- err = walk_pud_range(pgd, addr, next, walk);
+ err = walk_p4d_range(pgd, addr, next, walk);
if (err)
break;
} while (pgd++, addr = next, addr != end);
pgd_clear(pgd);
}
+void p4d_clear_bad(p4d_t *p4d)
+{
+ p4d_ERROR(*p4d);
+ p4d_clear(p4d);
+}
+
void pud_clear_bad(pud_t *pud)
{
pud_ERROR(*pud);
#include <linux/mm.h>
#include <linux/uio.h>
#include <linux/sched.h>
+#include <linux/sched/mm.h>
#include <linux/highmem.h>
#include <linux/ptrace.h>
#include <linux/slab.h>
*/
#include <linux/mm.h>
+#include <linux/sched/mm.h>
+#include <linux/sched/task.h>
#include <linux/pagemap.h>
#include <linux/swap.h>
#include <linux/swapops.h>
pmd_t *mm_find_pmd(struct mm_struct *mm, unsigned long address)
{
pgd_t *pgd;
+ p4d_t *p4d;
pud_t *pud;
pmd_t *pmd = NULL;
pmd_t pmde;
if (!pgd_present(*pgd))
goto out;
- pud = pud_offset(pgd, address);
+ p4d = p4d_offset(pgd, address);
+ if (!p4d_present(*p4d))
+ goto out;
+
+ pud = pud_offset(p4d, address);
if (!pud_present(*pud))
goto out;
}
while (page_vma_mapped_walk(&pvmw)) {
- subpage = page - page_to_pfn(page) + pte_pfn(*pvmw.pte);
- address = pvmw.address;
-
- /* Unexpected PMD-mapped THP? */
- VM_BUG_ON_PAGE(!pvmw.pte, page);
-
/*
* If the page is mlock()d, we cannot swap it out.
* If it's recently referenced (perhaps page_referenced
continue;
}
+ /* Unexpected PMD-mapped THP? */
+ VM_BUG_ON_PAGE(!pvmw.pte, page);
+
+ subpage = page - page_to_pfn(page) + pte_pfn(*pvmw.pte);
+ address = pvmw.address;
+
+
if (!(flags & TTU_IGNORE_ACCESS)) {
if (ptep_clear_flush_young_notify(vma, address,
pvmw.pte)) {
#include <linux/pagemap.h>
#include <linux/file.h>
#include <linux/mm.h>
+#include <linux/sched/signal.h>
#include <linux/export.h>
#include <linux/swap.h>
#include <linux/uio.h>
}
EXPORT_SYMBOL_GPL(shmem_truncate_range);
-static int shmem_getattr(struct vfsmount *mnt, struct dentry *dentry,
- struct kstat *stat)
+static int shmem_getattr(const struct path *path, struct kstat *stat,
+ u32 request_mask, unsigned int query_flags)
{
- struct inode *inode = dentry->d_inode;
+ struct inode *inode = path->dentry->d_inode;
struct shmem_inode_info *info = SHMEM_I(inode);
if (info->alloced - info->swapped != inode->i_mapping->nrpages) {
#include <linux/kmemcheck.h>
#include <linux/memory.h>
#include <linux/prefetch.h>
+#include <linux/sched/task_stack.h>
#include <net/sock.h>
return pmd;
}
-pud_t * __meminit vmemmap_pud_populate(pgd_t *pgd, unsigned long addr, int node)
+pud_t * __meminit vmemmap_pud_populate(p4d_t *p4d, unsigned long addr, int node)
{
- pud_t *pud = pud_offset(pgd, addr);
+ pud_t *pud = pud_offset(p4d, addr);
if (pud_none(*pud)) {
void *p = vmemmap_alloc_block(PAGE_SIZE, node);
if (!p)
return pud;
}
+p4d_t * __meminit vmemmap_p4d_populate(pgd_t *pgd, unsigned long addr, int node)
+{
+ p4d_t *p4d = p4d_offset(pgd, addr);
+ if (p4d_none(*p4d)) {
+ void *p = vmemmap_alloc_block(PAGE_SIZE, node);
+ if (!p)
+ return NULL;
+ p4d_populate(&init_mm, p4d, p);
+ }
+ return p4d;
+}
+
pgd_t * __meminit vmemmap_pgd_populate(unsigned long addr, int node)
{
pgd_t *pgd = pgd_offset_k(addr);
{
unsigned long addr = start;
pgd_t *pgd;
+ p4d_t *p4d;
pud_t *pud;
pmd_t *pmd;
pte_t *pte;
pgd = vmemmap_pgd_populate(addr, node);
if (!pgd)
return -ENOMEM;
- pud = vmemmap_pud_populate(pgd, addr, node);
+ p4d = vmemmap_p4d_populate(pgd, addr, node);
+ if (!p4d)
+ return -ENOMEM;
+ pud = vmemmap_pud_populate(p4d, addr, node);
if (!pud)
return -ENOMEM;
pmd = vmemmap_pmd_populate(pud, addr, node);
*/
#include <linux/mm.h>
+#include <linux/sched/mm.h>
+#include <linux/sched/task.h>
#include <linux/hugetlb.h>
#include <linux/mman.h>
#include <linux/slab.h>
return 0;
}
-static inline int unuse_pud_range(struct vm_area_struct *vma, pgd_t *pgd,
+static inline int unuse_pud_range(struct vm_area_struct *vma, p4d_t *p4d,
unsigned long addr, unsigned long end,
swp_entry_t entry, struct page *page)
{
unsigned long next;
int ret;
- pud = pud_offset(pgd, addr);
+ pud = pud_offset(p4d, addr);
do {
next = pud_addr_end(addr, end);
if (pud_none_or_clear_bad(pud))
return 0;
}
+static inline int unuse_p4d_range(struct vm_area_struct *vma, pgd_t *pgd,
+ unsigned long addr, unsigned long end,
+ swp_entry_t entry, struct page *page)
+{
+ p4d_t *p4d;
+ unsigned long next;
+ int ret;
+
+ p4d = p4d_offset(pgd, addr);
+ do {
+ next = p4d_addr_end(addr, end);
+ if (p4d_none_or_clear_bad(p4d))
+ continue;
+ ret = unuse_pud_range(vma, p4d, addr, next, entry, page);
+ if (ret)
+ return ret;
+ } while (p4d++, addr = next, addr != end);
+ return 0;
+}
+
static int unuse_vma(struct vm_area_struct *vma,
swp_entry_t entry, struct page *page)
{
next = pgd_addr_end(addr, end);
if (pgd_none_or_clear_bad(pgd))
continue;
- ret = unuse_pud_range(vma, pgd, addr, next, entry, page);
+ ret = unuse_p4d_range(vma, pgd, addr, next, entry, page);
if (ret)
return ret;
} while (pgd++, addr = next, addr != end);
#include <linux/mm.h>
#include <linux/slab.h>
+#include <linux/sched.h>
+#include <linux/sched/task.h>
+#include <linux/sched/task_stack.h>
#include <asm/sections.h>
enum {
*/
#include <linux/mm.h>
+#include <linux/sched/signal.h>
#include <linux/pagemap.h>
#include <linux/rmap.h>
#include <linux/swap.h>
static pmd_t *mm_alloc_pmd(struct mm_struct *mm, unsigned long address)
{
pgd_t *pgd;
+ p4d_t *p4d;
pud_t *pud;
- pmd_t *pmd = NULL;
pgd = pgd_offset(mm, address);
- pud = pud_alloc(mm, pgd, address);
- if (pud)
- /*
- * Note that we didn't run this because the pmd was
- * missing, the *pmd may be already established and in
- * turn it may also be a trans_huge_pmd.
- */
- pmd = pmd_alloc(mm, pud, address);
- return pmd;
+ p4d = p4d_alloc(mm, pgd, address);
+ if (!p4d)
+ return NULL;
+ pud = pud_alloc(mm, p4d, address);
+ if (!pud)
+ return NULL;
+ /*
+ * Note that we didn't run this because the pmd was
+ * missing, the *pmd may be already established and in
+ * turn it may also be a trans_huge_pmd.
+ */
+ return pmd_alloc(mm, pud, address);
}
#ifdef CONFIG_HUGETLB_PAGE
#include <linux/export.h>
#include <linux/err.h>
#include <linux/sched.h>
+#include <linux/sched/mm.h>
+#include <linux/sched/task_stack.h>
#include <linux/security.h>
#include <linux/swap.h>
#include <linux/swapops.h>
/*
* Copyright (C) 2014 Davidlohr Bueso.
*/
-#include <linux/sched.h>
+#include <linux/sched/signal.h>
+#include <linux/sched/task.h>
#include <linux/mm.h>
#include <linux/vmacache.h>
void vmacache_update(unsigned long addr, struct vm_area_struct *newvma)
{
if (vmacache_valid_mm(newvma->vm_mm))
- current->vmacache[VMACACHE_HASH(addr)] = newvma;
+ current->vmacache.vmas[VMACACHE_HASH(addr)] = newvma;
}
static bool vmacache_valid(struct mm_struct *mm)
return false;
curr = current;
- if (mm->vmacache_seqnum != curr->vmacache_seqnum) {
+ if (mm->vmacache_seqnum != curr->vmacache.seqnum) {
/*
* First attempt will always be invalid, initialize
* the new cache for this task here.
*/
- curr->vmacache_seqnum = mm->vmacache_seqnum;
+ curr->vmacache.seqnum = mm->vmacache_seqnum;
vmacache_flush(curr);
return false;
}
return NULL;
for (i = 0; i < VMACACHE_SIZE; i++) {
- struct vm_area_struct *vma = current->vmacache[i];
+ struct vm_area_struct *vma = current->vmacache.vmas[i];
if (!vma)
continue;
return NULL;
for (i = 0; i < VMACACHE_SIZE; i++) {
- struct vm_area_struct *vma = current->vmacache[i];
+ struct vm_area_struct *vma = current->vmacache.vmas[i];
if (vma && vma->vm_start == start && vma->vm_end == end) {
count_vm_vmacache_event(VMACACHE_FIND_HITS);
#include <linux/mm.h>
#include <linux/module.h>
#include <linux/highmem.h>
-#include <linux/sched.h>
+#include <linux/sched/signal.h>
#include <linux/slab.h>
#include <linux/spinlock.h>
#include <linux/interrupt.h>
} while (pmd++, addr = next, addr != end);
}
-static void vunmap_pud_range(pgd_t *pgd, unsigned long addr, unsigned long end)
+static void vunmap_pud_range(p4d_t *p4d, unsigned long addr, unsigned long end)
{
pud_t *pud;
unsigned long next;
- pud = pud_offset(pgd, addr);
+ pud = pud_offset(p4d, addr);
do {
next = pud_addr_end(addr, end);
if (pud_clear_huge(pud))
} while (pud++, addr = next, addr != end);
}
+static void vunmap_p4d_range(pgd_t *pgd, unsigned long addr, unsigned long end)
+{
+ p4d_t *p4d;
+ unsigned long next;
+
+ p4d = p4d_offset(pgd, addr);
+ do {
+ next = p4d_addr_end(addr, end);
+ if (p4d_clear_huge(p4d))
+ continue;
+ if (p4d_none_or_clear_bad(p4d))
+ continue;
+ vunmap_pud_range(p4d, addr, next);
+ } while (p4d++, addr = next, addr != end);
+}
+
static void vunmap_page_range(unsigned long addr, unsigned long end)
{
pgd_t *pgd;
next = pgd_addr_end(addr, end);
if (pgd_none_or_clear_bad(pgd))
continue;
- vunmap_pud_range(pgd, addr, next);
+ vunmap_p4d_range(pgd, addr, next);
} while (pgd++, addr = next, addr != end);
}
return 0;
}
-static int vmap_pud_range(pgd_t *pgd, unsigned long addr,
+static int vmap_pud_range(p4d_t *p4d, unsigned long addr,
unsigned long end, pgprot_t prot, struct page **pages, int *nr)
{
pud_t *pud;
unsigned long next;
- pud = pud_alloc(&init_mm, pgd, addr);
+ pud = pud_alloc(&init_mm, p4d, addr);
if (!pud)
return -ENOMEM;
do {
return 0;
}
+static int vmap_p4d_range(pgd_t *pgd, unsigned long addr,
+ unsigned long end, pgprot_t prot, struct page **pages, int *nr)
+{
+ p4d_t *p4d;
+ unsigned long next;
+
+ p4d = p4d_alloc(&init_mm, pgd, addr);
+ if (!p4d)
+ return -ENOMEM;
+ do {
+ next = p4d_addr_end(addr, end);
+ if (vmap_pud_range(p4d, addr, next, prot, pages, nr))
+ return -ENOMEM;
+ } while (p4d++, addr = next, addr != end);
+ return 0;
+}
+
/*
* Set up page tables in kva (addr, end). The ptes shall have prot "prot", and
* will have pfns corresponding to the "pages" array.
pgd = pgd_offset_k(addr);
do {
next = pgd_addr_end(addr, end);
- err = vmap_pud_range(pgd, addr, next, prot, pages, &nr);
+ err = vmap_p4d_range(pgd, addr, next, prot, pages, &nr);
if (err)
return err;
} while (pgd++, addr = next, addr != end);
unsigned long addr = (unsigned long) vmalloc_addr;
struct page *page = NULL;
pgd_t *pgd = pgd_offset_k(addr);
+ p4d_t *p4d;
+ pud_t *pud;
+ pmd_t *pmd;
+ pte_t *ptep, pte;
/*
* XXX we might need to change this if we add VIRTUAL_BUG_ON for
*/
VIRTUAL_BUG_ON(!is_vmalloc_or_module_addr(vmalloc_addr));
- if (!pgd_none(*pgd)) {
- pud_t *pud = pud_offset(pgd, addr);
- if (!pud_none(*pud)) {
- pmd_t *pmd = pmd_offset(pud, addr);
- if (!pmd_none(*pmd)) {
- pte_t *ptep, pte;
-
- ptep = pte_offset_map(pmd, addr);
- pte = *ptep;
- if (pte_present(pte))
- page = pte_page(pte);
- pte_unmap(ptep);
- }
- }
- }
+ if (pgd_none(*pgd))
+ return NULL;
+ p4d = p4d_offset(pgd, addr);
+ if (p4d_none(*p4d))
+ return NULL;
+ pud = pud_offset(p4d, addr);
+ if (pud_none(*pud))
+ return NULL;
+ pmd = pmd_offset(pud, addr);
+ if (pmd_none(*pmd))
+ return NULL;
+
+ ptep = pte_offset_map(pmd, addr);
+ pte = *ptep;
+ if (pte_present(pte))
+ page = pte_page(pte);
+ pte_unmap(ptep);
return page;
}
EXPORT_SYMBOL(vmalloc_to_page);
#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
#include <linux/mm.h>
+#include <linux/sched/mm.h>
#include <linux/module.h>
#include <linux/gfp.h>
#include <linux/kernel_stat.h>
"thp_split_page_failed",
"thp_deferred_split_page",
"thp_split_pmd",
+#ifdef CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD
+ "thp_split_pud",
+#endif
"thp_zero_page_alloc",
"thp_zero_page_alloc_failed",
#endif
#include <linux/module.h>
#include <linux/kernel.h>
#include <linux/sched.h>
+#include <linux/magic.h>
#include <linux/bitops.h>
#include <linux/errno.h>
#include <linux/highmem.h>
#include <linux/idr.h>
#include <linux/mutex.h>
#include <linux/slab.h>
-#include <linux/sched.h>
+#include <linux/sched/signal.h>
#include <linux/uaccess.h>
#include <linux/uio.h>
#include <net/9p/9p.h>
EXPORT_SYMBOL(p9_client_begin_disconnect);
struct p9_fid *p9_client_attach(struct p9_client *clnt, struct p9_fid *afid,
- char *uname, kuid_t n_uname, char *aname)
+ const char *uname, kuid_t n_uname, const char *aname)
{
int err = 0;
struct p9_req_t *req;
EXPORT_SYMBOL(p9_client_attach);
struct p9_fid *p9_client_walk(struct p9_fid *oldfid, uint16_t nwname,
- char **wnames, int clone)
+ const unsigned char * const *wnames, int clone)
{
int err;
struct p9_client *clnt;
}
EXPORT_SYMBOL(p9_client_open);
-int p9_client_create_dotl(struct p9_fid *ofid, char *name, u32 flags, u32 mode,
+int p9_client_create_dotl(struct p9_fid *ofid, const char *name, u32 flags, u32 mode,
kgid_t gid, struct p9_qid *qid)
{
int err = 0;
}
EXPORT_SYMBOL(p9_client_create_dotl);
-int p9_client_fcreate(struct p9_fid *fid, char *name, u32 perm, int mode,
+int p9_client_fcreate(struct p9_fid *fid, const char *name, u32 perm, int mode,
char *extension)
{
int err;
}
EXPORT_SYMBOL(p9_client_fcreate);
-int p9_client_symlink(struct p9_fid *dfid, char *name, char *symtgt, kgid_t gid,
- struct p9_qid *qid)
+int p9_client_symlink(struct p9_fid *dfid, const char *name,
+ const char *symtgt, kgid_t gid, struct p9_qid *qid)
{
int err = 0;
struct p9_client *clnt;
}
EXPORT_SYMBOL(p9_client_symlink);
-int p9_client_link(struct p9_fid *dfid, struct p9_fid *oldfid, char *newname)
+int p9_client_link(struct p9_fid *dfid, struct p9_fid *oldfid, const char *newname)
{
struct p9_client *clnt;
struct p9_req_t *req;
}
EXPORT_SYMBOL(p9_client_readdir);
-int p9_client_mknod_dotl(struct p9_fid *fid, char *name, int mode,
+int p9_client_mknod_dotl(struct p9_fid *fid, const char *name, int mode,
dev_t rdev, kgid_t gid, struct p9_qid *qid)
{
int err;
}
EXPORT_SYMBOL(p9_client_mknod_dotl);
-int p9_client_mkdir_dotl(struct p9_fid *fid, char *name, int mode,
+int p9_client_mkdir_dotl(struct p9_fid *fid, const char *name, int mode,
kgid_t gid, struct p9_qid *qid)
{
int err;
#include <linux/errno.h> /* error codes */
#include <linux/capability.h>
#include <linux/mm.h>
-#include <linux/sched.h>
+#include <linux/sched/signal.h>
#include <linux/time.h> /* struct timeval */
#include <linux/skbuff.h>
#include <linux/bitops.h>
#include <linux/kernel.h> /* printk */
#include <linux/skbuff.h>
#include <linux/wait.h>
-#include <linux/sched.h> /* jiffies and HZ */
+#include <linux/sched/signal.h>
#include <linux/fcntl.h> /* O_NONBLOCK */
#include <linux/init.h>
#include <linux/atm.h> /* ATM stuff */
#include <linux/socket.h>
#include <linux/in.h>
#include <linux/kernel.h>
-#include <linux/sched.h>
+#include <linux/sched/signal.h>
#include <linux/timer.h>
#include <linux/string.h>
#include <linux/sockios.h>
spin_unlock_bh(&chain->lock);
err:
- if (!ret)
+ if (!ret) {
kfree(frag_entry_new);
+ kfree_skb(skb);
+ }
return ret;
}
*
* There are three possible outcomes: 1) Packet is merged: Return true and
* set *skb to merged packet; 2) Packet is buffered: Return true and set *skb
- * to NULL; 3) Error: Return false and leave skb as is.
+ * to NULL; 3) Error: Return false and free skb.
*
* Return: true when packet is merged or buffered, false when skb is not not
* used.
goto out_err;
out:
- *skb = skb_out;
ret = true;
out_err:
+ *skb = skb_out;
return ret;
}
/* Eat and send fragments from the tail of skb */
while (skb->len > max_fragment_size) {
+ /* The initial check in this function should cover this case */
+ if (unlikely(frag_header.no == BATADV_FRAG_MAX_FRAGMENTS - 1)) {
+ ret = -EINVAL;
+ goto put_primary_if;
+ }
+
skb_fragment = batadv_frag_create(skb, &frag_header, mtu);
if (!skb_fragment) {
ret = -ENOMEM;
}
frag_header.no++;
-
- /* The initial check in this function should cover this case */
- if (frag_header.no == BATADV_FRAG_MAX_FRAGMENTS - 1) {
- ret = -EINVAL;
- goto put_primary_if;
- }
}
/* Make room for the fragment header. */
struct rcu_head rcu;
};
-DECLARE_EWMA(throughput, 1024, 8)
+DECLARE_EWMA(throughput, 10, 8)
/**
* struct batadv_hardif_neigh_node_bat_v - B.A.T.M.A.N. V private neighbor
#include <linux/module.h>
#include <linux/debugfs.h>
#include <linux/stringify.h>
+#include <linux/sched/signal.h>
+
#include <asm/ioctls.h>
#include <net/bluetooth/bluetooth.h>
#include <linux/types.h>
#include <linux/errno.h>
#include <linux/kernel.h>
-#include <linux/sched.h>
+#include <linux/sched/signal.h>
#include <linux/slab.h>
#include <linux/poll.h>
#include <linux/fcntl.h>
SOFTWARE IS DISCLAIMED.
*/
+#include <linux/sched/signal.h>
+
#include <net/bluetooth/bluetooth.h>
#include <net/bluetooth/hci_core.h>
#include <net/bluetooth/mgmt.h>
#include <linux/module.h>
#include <linux/export.h>
+#include <linux/sched/signal.h>
#include <net/bluetooth/bluetooth.h>
#include <net/bluetooth/hci_core.h>
#include <linux/export.h>
#include <linux/debugfs.h>
+#include <linux/sched/signal.h>
#include <net/bluetooth/bluetooth.h>
#include <net/bluetooth/hci_core.h>
#include <linux/module.h>
#include <linux/debugfs.h>
#include <linux/seq_file.h>
+#include <linux/sched/signal.h>
#include <net/bluetooth/bluetooth.h>
#include <net/bluetooth/hci_core.h>
/* Do not flood unicast traffic to ports that turn it off */
if (pkt_type == BR_PKT_UNICAST && !(p->flags & BR_FLOOD))
continue;
+ /* Do not flood if mc off, except for traffic we originate */
if (pkt_type == BR_PKT_MULTICAST &&
- !(p->flags & BR_MCAST_FLOOD))
+ !(p->flags & BR_MCAST_FLOOD) && skb->dev != br->dev)
continue;
/* Do not flood to ports that enable proxy ARP */
#include <linux/rtnetlink.h>
#include <linux/spinlock.h>
#include <linux/times.h>
+#include <linux/sched/signal.h>
#include "br_private.h"
#include <linux/if_bridge.h>
#include <linux/rtnetlink.h>
#include <linux/spinlock.h>
+#include <linux/sched/signal.h>
#include "br_private.h"
RCU_INIT_POINTER(p->vlgrp, NULL);
synchronize_rcu();
vlan_tunnel_deinit(vg);
-err_vlan_enabled:
err_tunnel_init:
rhashtable_destroy(&vg->vlan_hash);
err_rhtbl:
+err_vlan_enabled:
kfree(vg);
goto out;
#include <linux/fs.h>
#include <linux/init.h>
#include <linux/module.h>
-#include <linux/sched.h>
+#include <linux/sched/signal.h>
#include <linux/spinlock.h>
#include <linux/mutex.h>
#include <linux/list.h>
Opt_osdkeepalivetimeout,
Opt_mount_timeout,
Opt_osd_idle_ttl,
+ Opt_osd_request_timeout,
Opt_last_int,
/* int args above */
Opt_fsid,
{Opt_osdkeepalivetimeout, "osdkeepalive=%d"},
{Opt_mount_timeout, "mount_timeout=%d"},
{Opt_osd_idle_ttl, "osd_idle_ttl=%d"},
+ {Opt_osd_request_timeout, "osd_request_timeout=%d"},
/* int args above */
{Opt_fsid, "fsid=%s"},
{Opt_name, "name=%s"},
opt->osd_keepalive_timeout = CEPH_OSD_KEEPALIVE_DEFAULT;
opt->mount_timeout = CEPH_MOUNT_TIMEOUT_DEFAULT;
opt->osd_idle_ttl = CEPH_OSD_IDLE_TTL_DEFAULT;
+ opt->osd_request_timeout = CEPH_OSD_REQUEST_TIMEOUT_DEFAULT;
/* get mon ip(s) */
/* ip1[:port1][,ip2[:port2]...] */
}
opt->mount_timeout = msecs_to_jiffies(intval * 1000);
break;
+ case Opt_osd_request_timeout:
+ /* 0 is "wait forever" (i.e. infinite timeout) */
+ if (intval < 0 || intval > INT_MAX / 1000) {
+ pr_err("osd_request_timeout out of range\n");
+ err = -EINVAL;
+ goto out;
+ }
+ opt->osd_request_timeout = msecs_to_jiffies(intval * 1000);
+ break;
case Opt_share:
opt->flags &= ~CEPH_OPT_NOSHARE;
if (opt->osd_keepalive_timeout != CEPH_OSD_KEEPALIVE_DEFAULT)
seq_printf(m, "osdkeepalivetimeout=%d,",
jiffies_to_msecs(opt->osd_keepalive_timeout) / 1000);
+ if (opt->osd_request_timeout != CEPH_OSD_REQUEST_TIMEOUT_DEFAULT)
+ seq_printf(m, "osd_request_timeout=%d,",
+ jiffies_to_msecs(opt->osd_request_timeout) / 1000);
/* drop redundant comma */
if (m->count != pos)
#include <crypto/aes.h>
#include <crypto/skcipher.h>
#include <linux/key-type.h>
+#include <linux/sched/mm.h>
#include <keys/ceph-type.h>
#include <keys/user-type.h>
struct msghdr msg = { .msg_flags = MSG_DONTWAIT | MSG_NOSIGNAL };
int r;
- r = kernel_recvmsg(sock, &msg, &iov, 1, len, msg.msg_flags);
+ iov_iter_kvec(&msg.msg_iter, READ | ITER_KVEC, &iov, 1, len);
+ r = sock_recvmsg(sock, &msg, msg.msg_flags);
if (r == -EAGAIN)
r = 0;
return r;
static int ceph_tcp_recvpage(struct socket *sock, struct page *page,
int page_offset, size_t length)
{
- void *kaddr;
- int ret;
+ struct bio_vec bvec = {
+ .bv_page = page,
+ .bv_offset = page_offset,
+ .bv_len = length
+ };
+ struct msghdr msg = { .msg_flags = MSG_DONTWAIT | MSG_NOSIGNAL };
+ int r;
BUG_ON(page_offset + length > PAGE_SIZE);
-
- kaddr = kmap(page);
- BUG_ON(!kaddr);
- ret = ceph_tcp_recvmsg(sock, kaddr + page_offset, length);
- kunmap(page);
-
- return ret;
+ iov_iter_bvec(&msg.msg_iter, READ | ITER_BVEC, &bvec, 1, length);
+ r = sock_recvmsg(sock, &msg, msg.msg_flags);
+ if (r == -EAGAIN)
+ r = 0;
+ return r;
}
/*
static int ceph_tcp_sendpage(struct socket *sock, struct page *page,
int offset, size_t size, bool more)
{
+ struct msghdr msg = { .msg_flags = MSG_DONTWAIT | MSG_NOSIGNAL };
+ struct bio_vec bvec;
int ret;
- struct kvec iov;
/* sendpage cannot properly handle pages with page_count == 0,
* we need to fallback to sendmsg if that's the case */
if (page_count(page) >= 1)
return __ceph_tcp_sendpage(sock, page, offset, size, more);
- iov.iov_base = kmap(page) + offset;
- iov.iov_len = size;
- ret = ceph_tcp_sendmsg(sock, &iov, 1, size, more);
- kunmap(page);
+ bvec.bv_page = page;
+ bvec.bv_offset = offset;
+ bvec.bv_len = size;
+
+ if (more)
+ msg.msg_flags |= MSG_MORE;
+ else
+ msg.msg_flags |= MSG_EOR; /* superfluous, but what the hell */
+
+ iov_iter_bvec(&msg.msg_iter, WRITE | ITER_BVEC, &bvec, 1, size);
+ ret = sock_sendmsg(sock, &msg);
+ if (ret == -EAGAIN)
+ ret = 0;
return ret;
}
req->r_flags |= CEPH_OSD_FLAG_ONDISK;
atomic_inc(&req->r_osdc->num_requests);
+
+ req->r_start_stamp = jiffies;
}
static void submit_request(struct ceph_osd_request *req, bool wrlocked)
ceph_osdc_put_request(req);
}
+static void abort_request(struct ceph_osd_request *req, int err)
+{
+ dout("%s req %p tid %llu err %d\n", __func__, req, req->r_tid, err);
+
+ cancel_map_check(req);
+ complete_request(req, err);
+}
+
static void check_pool_dne(struct ceph_osd_request *req)
{
struct ceph_osd_client *osdc = req->r_osdc;
container_of(work, struct ceph_osd_client, timeout_work.work);
struct ceph_options *opts = osdc->client->options;
unsigned long cutoff = jiffies - opts->osd_keepalive_timeout;
+ unsigned long expiry_cutoff = jiffies - opts->osd_request_timeout;
LIST_HEAD(slow_osds);
struct rb_node *n, *p;
struct ceph_osd *osd = rb_entry(n, struct ceph_osd, o_node);
bool found = false;
- for (p = rb_first(&osd->o_requests); p; p = rb_next(p)) {
+ for (p = rb_first(&osd->o_requests); p; ) {
struct ceph_osd_request *req =
rb_entry(p, struct ceph_osd_request, r_node);
+ p = rb_next(p); /* abort_request() */
+
if (time_before(req->r_stamp, cutoff)) {
dout(" req %p tid %llu on osd%d is laggy\n",
req, req->r_tid, osd->o_osd);
found = true;
}
+ if (opts->osd_request_timeout &&
+ time_before(req->r_start_stamp, expiry_cutoff)) {
+ pr_err_ratelimited("tid %llu on osd%d timeout\n",
+ req->r_tid, osd->o_osd);
+ abort_request(req, -ETIMEDOUT);
+ }
}
for (p = rb_first(&osd->o_linger_requests); p; p = rb_next(p)) {
struct ceph_osd_linger_request *lreq =
list_move_tail(&osd->o_keepalive_item, &slow_osds);
}
+ if (opts->osd_request_timeout) {
+ for (p = rb_first(&osdc->homeless_osd.o_requests); p; ) {
+ struct ceph_osd_request *req =
+ rb_entry(p, struct ceph_osd_request, r_node);
+
+ p = rb_next(p); /* abort_request() */
+
+ if (time_before(req->r_start_stamp, expiry_cutoff)) {
+ pr_err_ratelimited("tid %llu on osd%d timeout\n",
+ req->r_tid, osdc->homeless_osd.o_osd);
+ abort_request(req, -ETIMEDOUT);
+ }
+ }
+ }
+
if (atomic_read(&osdc->num_homeless) || !list_empty(&slow_osds))
maybe_request_map(osdc);
dout("crush decode tunable chooseleaf_stable = %d\n",
c->chooseleaf_stable);
- crush_finalize(c);
-
done:
+ crush_finalize(c);
dout("crush_decode success\n");
return c;
if ((map->osd_state[osd] & CEPH_OSD_EXISTS) &&
(xorstate & CEPH_OSD_EXISTS)) {
pr_info("osd%d does not exist\n", osd);
- map->osd_weight[osd] = CEPH_OSD_IN;
ret = set_primary_affinity(map, osd,
CEPH_OSD_DEFAULT_PRIMARY_AFFINITY);
if (ret)
static struct static_key netstamp_needed __read_mostly;
#ifdef HAVE_JUMP_LABEL
static atomic_t netstamp_needed_deferred;
+static atomic_t netstamp_wanted;
static void netstamp_clear(struct work_struct *work)
{
int deferred = atomic_xchg(&netstamp_needed_deferred, 0);
+ int wanted;
- while (deferred--)
- static_key_slow_dec(&netstamp_needed);
+ wanted = atomic_add_return(deferred, &netstamp_wanted);
+ if (wanted > 0)
+ static_key_enable(&netstamp_needed);
+ else
+ static_key_disable(&netstamp_needed);
}
static DECLARE_WORK(netstamp_work, netstamp_clear);
#endif
void net_enable_timestamp(void)
{
+#ifdef HAVE_JUMP_LABEL
+ int wanted;
+
+ while (1) {
+ wanted = atomic_read(&netstamp_wanted);
+ if (wanted <= 0)
+ break;
+ if (atomic_cmpxchg(&netstamp_wanted, wanted, wanted + 1) == wanted)
+ return;
+ }
+ atomic_inc(&netstamp_needed_deferred);
+ schedule_work(&netstamp_work);
+#else
static_key_slow_inc(&netstamp_needed);
+#endif
}
EXPORT_SYMBOL(net_enable_timestamp);
void net_disable_timestamp(void)
{
#ifdef HAVE_JUMP_LABEL
- /* net_disable_timestamp() can be called from non process context */
- atomic_inc(&netstamp_needed_deferred);
+ int wanted;
+
+ while (1) {
+ wanted = atomic_read(&netstamp_wanted);
+ if (wanted <= 1)
+ break;
+ if (atomic_cmpxchg(&netstamp_wanted, wanted, wanted - 1) == wanted)
+ return;
+ }
+ atomic_dec(&netstamp_needed_deferred);
schedule_work(&netstamp_work);
#else
static_key_slow_dec(&netstamp_needed);
}
EXPORT_SYMBOL(__napi_schedule);
+/**
+ * napi_schedule_prep - check if napi can be scheduled
+ * @n: napi context
+ *
+ * Test if NAPI routine is already running, and if not mark
+ * it as running. This is used as a condition variable
+ * insure only one NAPI poll instance runs. We also make
+ * sure there is no pending NAPI disable.
+ */
+bool napi_schedule_prep(struct napi_struct *n)
+{
+ unsigned long val, new;
+
+ do {
+ val = READ_ONCE(n->state);
+ if (unlikely(val & NAPIF_STATE_DISABLE))
+ return false;
+ new = val | NAPIF_STATE_SCHED;
+
+ /* Sets STATE_MISSED bit if STATE_SCHED was already set
+ * This was suggested by Alexander Duyck, as compiler
+ * emits better code than :
+ * if (val & NAPIF_STATE_SCHED)
+ * new |= NAPIF_STATE_MISSED;
+ */
+ new |= (val & NAPIF_STATE_SCHED) / NAPIF_STATE_SCHED *
+ NAPIF_STATE_MISSED;
+ } while (cmpxchg(&n->state, val, new) != val);
+
+ return !(val & NAPIF_STATE_SCHED);
+}
+EXPORT_SYMBOL(napi_schedule_prep);
+
/**
* __napi_schedule_irqoff - schedule for receive
* @n: entry to schedule
bool napi_complete_done(struct napi_struct *n, int work_done)
{
- unsigned long flags;
+ unsigned long flags, val, new;
/*
* 1) Don't let napi dequeue from the cpu poll list
list_del_init(&n->poll_list);
local_irq_restore(flags);
}
- WARN_ON_ONCE(!test_and_clear_bit(NAPI_STATE_SCHED, &n->state));
+
+ do {
+ val = READ_ONCE(n->state);
+
+ WARN_ON_ONCE(!(val & NAPIF_STATE_SCHED));
+
+ new = val & ~(NAPIF_STATE_MISSED | NAPIF_STATE_SCHED);
+
+ /* If STATE_MISSED was set, leave STATE_SCHED set,
+ * because we will call napi->poll() one more time.
+ * This C code was suggested by Alexander Duyck to help gcc.
+ */
+ new |= (val & NAPIF_STATE_MISSED) / NAPIF_STATE_MISSED *
+ NAPIF_STATE_SCHED;
+ } while (cmpxchg(&n->state, val, new) != val);
+
+ if (unlikely(val & NAPIF_STATE_MISSED)) {
+ __napi_schedule(n);
+ return false;
+ }
+
return true;
}
EXPORT_SYMBOL(napi_complete_done);
{
int rc;
+ /* Busy polling means there is a high chance device driver hard irq
+ * could not grab NAPI_STATE_SCHED, and that NAPI_STATE_MISSED was
+ * set in napi_schedule_prep().
+ * Since we are about to call napi->poll() once more, we can safely
+ * clear NAPI_STATE_MISSED.
+ *
+ * Note: x86 could use a single "lock and ..." instruction
+ * to perform these two clear_bit()
+ */
+ clear_bit(NAPI_STATE_MISSED, &napi->state);
clear_bit(NAPI_STATE_IN_BUSY_POLL, &napi->state);
local_bh_disable();
struct napi_struct *napi;
napi = container_of(timer, struct napi_struct, timer);
- if (napi->gro_list)
- napi_schedule_irqoff(napi);
+
+ /* Note : we use a relaxed variant of napi_schedule_prep() not setting
+ * NAPI_STATE_MISSED, since we do not react to a device IRQ.
+ */
+ if (napi->gro_list && !napi_disable_pending(napi) &&
+ !test_and_set_bit(NAPI_STATE_SCHED, &napi->state))
+ __napi_schedule_irqoff(napi);
return HRTIMER_NORESTART;
}
#include <linux/vmalloc.h>
#include <linux/slab.h>
#include <linux/rtnetlink.h>
-#include <linux/sched.h>
+#include <linux/sched/signal.h>
#include <linux/net.h>
/*
#include <net/switchdev.h>
#include <linux/if_arp.h>
#include <linux/slab.h>
+#include <linux/sched/signal.h>
#include <linux/nsproxy.h>
#include <net/sock.h>
#include <net/net_namespace.h>
#include <linux/export.h>
#include <linux/user_namespace.h>
#include <linux/net_namespace.h>
+#include <linux/sched/task.h>
+
#include <net/sock.h>
#include <net/netlink.h>
#include <net/net_namespace.h>
#include <linux/slab.h>
#include <linux/cgroup.h>
#include <linux/fdtable.h>
+#include <linux/sched/task.h>
+
#include <net/cls_cgroup.h>
#include <net/sock.h>
#include <linux/cgroup.h>
#include <linux/rcupdate.h>
#include <linux/atomic.h>
+#include <linux/sched/task.h>
+
#include <net/rtnetlink.h>
#include <net/pkt_cls.h>
#include <net/sock.h>
#include <linux/capability.h>
#include <linux/errno.h>
#include <linux/sched.h>
+#include <linux/sched/user.h>
#include <linux/mm.h>
#include <linux/kernel.h>
#include <linux/stat.h>
is_charged = sk_filter_charge(newsk, filter);
if (unlikely(!is_charged || xfrm_sk_clone_policy(newsk, sk))) {
- /* It is still raw copy of parent, so invalidate
- * destructor and make plain sk_free() */
- newsk->sk_destruct = NULL;
- bh_unlock_sock(newsk);
- sk_free(newsk);
+ sk_free_unlock_clone(newsk);
newsk = NULL;
goto out;
}
}
EXPORT_SYMBOL_GPL(sk_clone_lock);
+void sk_free_unlock_clone(struct sock *sk)
+{
+ /* It is still raw copy of parent, so invalidate
+ * destructor and make plain sk_free() */
+ sk->sk_destruct = NULL;
+ bh_unlock_sock(sk);
+ sk_free(sk);
+}
+EXPORT_SYMBOL_GPL(sk_free_unlock_clone);
+
void sk_setup_caps(struct sock *sk, struct dst_entry *dst)
{
u32 max_segs = 1;
*/
#include <linux/module.h>
+#include <linux/sched/signal.h>
#include <linux/net.h>
#include <linux/signal.h>
#include <linux/tcp.h>
struct dccp_sock *dp = dccp_sk(sk);
struct dccp_skb_cb *dcb = DCCP_SKB_CB(skb);
const int old_state = sk->sk_state;
+ bool acceptable;
int queued = 0;
/*
*/
if (sk->sk_state == DCCP_LISTEN) {
if (dh->dccph_type == DCCP_PKT_REQUEST) {
- if (inet_csk(sk)->icsk_af_ops->conn_request(sk,
- skb) < 0)
+ /* It is possible that we process SYN packets from backlog,
+ * so we need to make sure to disable BH right there.
+ */
+ local_bh_disable();
+ acceptable = inet_csk(sk)->icsk_af_ops->conn_request(sk, skb) >= 0;
+ local_bh_enable();
+ if (!acceptable)
return 1;
consume_skb(skb);
return 0;
* Activate features: initialise CCIDs, sequence windows etc.
*/
if (dccp_feat_activate_values(newsk, &dreq->dreq_featneg)) {
- /* It is still raw copy of parent, so invalidate
- * destructor and make plain sk_free() */
- newsk->sk_destruct = NULL;
- sk_free(newsk);
+ sk_free_unlock_clone(newsk);
return NULL;
}
dccp_init_xmit_timers(newsk);
#include <linux/kernel.h>
#include <linux/skbuff.h>
#include <linux/slab.h>
+#include <linux/sched/signal.h>
#include <net/inet_sock.h>
#include <net/sock.h>
#include <linux/socket.h>
#include <linux/in.h>
#include <linux/kernel.h>
-#include <linux/sched.h>
+#include <linux/sched/signal.h>
#include <linux/timer.h>
#include <linux/string.h>
#include <linux/sockios.h>
#include <linux/module.h>
#include <linux/slab.h>
+#include <linux/cred.h>
#include <linux/dns_resolver.h>
#include <linux/err.h>
+
#include <keys/dns_resolver-type.h>
#include <keys/user-type.h>
const char *options, char **_result, time64_t *_expiry)
{
struct key *rkey;
- const struct user_key_payload *upayload;
+ struct user_key_payload *upayload;
const struct cred *saved_cred;
size_t typelen, desclen;
char *desc, *cp;
if (ret)
goto put;
- upayload = user_key_payload(rkey);
+ upayload = user_key_payload_locked(rkey);
len = upayload->datalen;
ret = -ENOMEM;
#include <linux/module.h>
#include <linux/types.h>
#include <linux/kernel.h>
+#include <linux/sched/signal.h>
#include <linux/string.h>
#include <linux/mm.h>
#include <linux/socket.h>
[RTA_ENCAP_TYPE] = { .type = NLA_U16 },
[RTA_ENCAP] = { .type = NLA_NESTED },
[RTA_UID] = { .type = NLA_U32 },
+ [RTA_MARK] = { .type = NLA_U32 },
};
static int rtm_to_fib_config(struct net *net, struct sk_buff *skb,
struct rtable *rt;
struct flowi4 fl4 = {};
__be32 saddr = iph->saddr;
- __u8 flags = skb->sk ? inet_sk_flowi_flags(skb->sk) : 0;
+ const struct sock *sk = skb_to_full_sk(skb);
+ __u8 flags = sk ? inet_sk_flowi_flags(sk) : 0;
struct net_device *dev = skb_dst(skb)->dev;
unsigned int hh_len;
fl4.daddr = iph->daddr;
fl4.saddr = saddr;
fl4.flowi4_tos = RT_TOS(iph->tos);
- fl4.flowi4_oif = skb->sk ? skb->sk->sk_bound_dev_if : 0;
+ fl4.flowi4_oif = sk ? sk->sk_bound_dev_if : 0;
if (!fl4.flowi4_oif)
fl4.flowi4_oif = l3mdev_master_ifindex(dev);
fl4.flowi4_mark = skb->mark;
xfrm_decode_session(skb, flowi4_to_flowi(&fl4), AF_INET) == 0) {
struct dst_entry *dst = skb_dst(skb);
skb_dst_set(skb, NULL);
- dst = xfrm_lookup(net, dst, flowi4_to_flowi(&fl4), skb->sk, 0);
+ dst = xfrm_lookup(net, dst, flowi4_to_flowi(&fl4), sk, 0);
if (IS_ERR(dst))
return PTR_ERR(dst);
skb_dst_set(skb, dst);
flags = (msg->msg_flags & MSG_DONTWAIT) ? O_NONBLOCK : 0;
err = __inet_stream_connect(sk->sk_socket, msg->msg_name,
msg->msg_namelen, flags, 1);
- inet->defer_connect = 0;
- *copied = tp->fastopen_req->copied;
- tcp_free_fastopen_req(tp);
+ /* fastopen_req could already be freed in __inet_stream_connect
+ * if the connection times out or gets rst
+ */
+ if (tp->fastopen_req) {
+ *copied = tp->fastopen_req->copied;
+ tcp_free_fastopen_req(tp);
+ inet->defer_connect = 0;
+ }
return err;
}
memset(&tp->rx_opt, 0, sizeof(tp->rx_opt));
__sk_dst_reset(sk);
+ /* Clean up fastopen related fields */
+ tcp_free_fastopen_req(tp);
+ inet->defer_connect = 0;
+
WARN_ON(inet->inet_num && !icsk->icsk_bind_hash);
sk->sk_error_report(sk);
#include <linux/kernel.h>
#include <linux/random.h>
#include <linux/module.h>
+#include <linux/sched/clock.h>
+
#include <net/tcp.h>
#define HYSTART_ACK_TRAIN 1
if (th->syn) {
if (th->fin)
goto discard;
- if (icsk->icsk_af_ops->conn_request(sk, skb) < 0)
- return 1;
+ /* It is possible that we process SYN packets from backlog,
+ * so we need to make sure to disable BH right there.
+ */
+ local_bh_disable();
+ acceptable = icsk->icsk_af_ops->conn_request(sk, skb) >= 0;
+ local_bh_enable();
+ if (!acceptable)
+ return 1;
consume_skb(skb);
return 0;
}
#include <linux/errno.h>
#include <linux/types.h>
#include <linux/kernel.h>
+#include <linux/sched/signal.h>
#include <linux/socket.h>
#include <linux/sockios.h>
#include <linux/net.h>
struct inet6_dev *idev = (struct inet6_dev *)ctl->extra1;
struct net *net = (struct net *)ctl->extra2;
+ if (!rtnl_trylock())
+ return restart_syscall();
+
ret = proc_dointvec(ctl, write, buffer, lenp, ppos);
if (write) {
new_val = *((int *)ctl->data);
- if (check_addr_gen_mode(new_val) < 0)
- return -EINVAL;
+ if (check_addr_gen_mode(new_val) < 0) {
+ ret = -EINVAL;
+ goto out;
+ }
/* request for default */
if (&net->ipv6.devconf_dflt->addr_gen_mode == ctl->data) {
/* request for individual net device */
} else {
if (!idev)
- return ret;
+ goto out;
- if (check_stable_privacy(idev, net, new_val) < 0)
- return -EINVAL;
+ if (check_stable_privacy(idev, net, new_val) < 0) {
+ ret = -EINVAL;
+ goto out;
+ }
if (idev->cnf.addr_gen_mode != new_val) {
idev->cnf.addr_gen_mode = new_val;
- rtnl_lock();
addrconf_dev_config(idev->dev);
- rtnl_unlock();
}
}
}
+out:
+ rtnl_unlock();
+
return ret;
}
hdr = ipv6_hdr(skb);
fhdr = (struct frag_hdr *)skb_transport_header(skb);
+ skb_orphan(skb);
fq = fq_find(net, fhdr->identification, user, &hdr->saddr, &hdr->daddr,
skb->dev ? skb->dev->ifindex : 0, ip6_frag_ecn(hdr));
if (fq == NULL) {
static int __ip6_del_rt_siblings(struct rt6_info *rt, struct fib6_config *cfg)
{
struct nl_info *info = &cfg->fc_nlinfo;
+ struct net *net = info->nl_net;
struct sk_buff *skb = NULL;
struct fib6_table *table;
- int err;
+ int err = -ENOENT;
+ if (rt == net->ipv6.ip6_null_entry)
+ goto out_put;
table = rt->rt6i_table;
write_lock_bh(&table->tb6_lock);
if (skb) {
u32 seq = info->nlh ? info->nlh->nlmsg_seq : 0;
- if (rt6_fill_node(info->nl_net, skb, rt,
+ if (rt6_fill_node(net, skb, rt,
NULL, NULL, 0, RTM_DELROUTE,
info->portid, seq, 0) < 0) {
kfree_skb(skb);
rt6i_siblings) {
err = fib6_del(sibling, info);
if (err)
- goto out;
+ goto out_unlock;
}
}
err = fib6_del(rt, info);
-out:
+out_unlock:
write_unlock_bh(&table->tb6_lock);
+out_put:
ip6_rt_put(rt);
if (skb) {
- rtnl_notify(skb, info->nl_net, info->portid, RTNLGRP_IPV6_ROUTE,
+ rtnl_notify(skb, net, info->portid, RTNLGRP_IPV6_ROUTE,
info->nlh, gfp_any());
}
return err;
[RTA_ENCAP] = { .type = NLA_NESTED },
[RTA_EXPIRES] = { .type = NLA_U32 },
[RTA_UID] = { .type = NLA_U32 },
+ [RTA_MARK] = { .type = NLA_U32 },
};
static int rtm_to_fib6_config(struct sk_buff *skb, struct nlmsghdr *nlh,
rt = (struct rt6_info *)ip6_route_output(net, NULL, &fl6);
}
+ if (rt == net->ipv6.ip6_null_entry) {
+ err = rt->dst.error;
+ ip6_rt_put(rt);
+ goto errout;
+ }
+
skb = alloc_skb(NLMSG_GOODSIZE, GFP_KERNEL);
if (!skb) {
ip6_rt_put(rt);
#include <linux/socket.h>
#include <linux/sockios.h>
#include <linux/slab.h>
+#include <linux/sched/signal.h>
#include <linux/init.h>
#include <linux/net.h>
#include <linux/irda.h>
#include <linux/module.h>
#include <linux/fs.h>
#include <linux/slab.h>
-#include <linux/sched.h>
+#include <linux/sched/signal.h>
#include <linux/seq_file.h>
#include <linux/termios.h>
#include <linux/tty.h>
* 2) as a control channel (write commands, read events)
*/
-#include <linux/sched.h>
+#include <linux/sched/signal.h>
#include <linux/slab.h>
+
#include "irnet_ppp.h" /* Private header */
/* Please put other headers in irnet.h - Thanks */
#include <linux/list.h>
#include <linux/errno.h>
#include <linux/kernel.h>
-#include <linux/sched.h>
+#include <linux/sched/signal.h>
#include <linux/slab.h>
#include <linux/skbuff.h>
#include <linux/init.h>
#include <linux/uaccess.h>
#include <linux/workqueue.h>
#include <linux/syscalls.h>
+#include <linux/sched/signal.h>
+
#include <net/kcm.h>
#include <net/netns/generic.h>
#include <net/sock.h>
#include <linux/rtnetlink.h>
#include <linux/init.h>
#include <linux/slab.h>
+#include <linux/sched/signal.h>
+
#include <net/llc.h>
#include <net/llc_sap.h>
#include <net/llc_pdu.h>
ht_dbg(sta->sdata,
"Rx BA session stop requested for %pM tid %u %s reason: %d\n",
sta->sta.addr, tid,
- initiator == WLAN_BACK_RECIPIENT ? "recipient" : "inititator",
+ initiator == WLAN_BACK_RECIPIENT ? "recipient" : "initiator",
(int)reason);
if (drv_ampdu_action(local, sta->sdata, ¶ms))
tid_agg_rx->timeout = timeout;
tid_agg_rx->stored_mpdu_num = 0;
tid_agg_rx->auto_seq = auto_seq;
+ tid_agg_rx->started = false;
tid_agg_rx->reorder_buf_filtered = 0;
status = WLAN_STATUS_SUCCESS;
bool downgraded;
};
-DECLARE_EWMA(beacon_signal, 16, 4)
+DECLARE_EWMA(beacon_signal, 4, 4)
struct ieee80211_if_managed {
struct timer_list timer;
#include <linux/gfp.h>
#include <linux/kernel.h>
#include <linux/random.h>
+#include <linux/rculist.h>
+
#include "ieee80211_i.h"
#include "rate.h"
#include "mesh.h"
break;
}
+ flush_delayed_work(&sdata->dec_tailroom_needed_wk);
drv_remove_interface(local, sdata);
}
* Copyright 2006-2007 Jiri Benc <jbenc@suse.cz>
* Copyright 2007-2010 Johannes Berg <johannes@sipsolutions.net>
* Copyright 2013-2014 Intel Mobile Communications GmbH
- * Copyright(c) 2015 - 2016 Intel Deutschland GmbH
+ * Copyright(c) 2015 - 2017 Intel Deutschland GmbH
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License version 2 as
buf_size = tid_agg_rx->buf_size;
head_seq_num = tid_agg_rx->head_seq_num;
+ /*
+ * If the current MPDU's SN is smaller than the SSN, it shouldn't
+ * be reordered.
+ */
+ if (unlikely(!tid_agg_rx->started)) {
+ if (ieee80211_sn_less(mpdu_seq_num, head_seq_num)) {
+ ret = false;
+ goto out;
+ }
+ tid_agg_rx->started = true;
+ }
+
/* frame with out of date sequence number */
if (ieee80211_sn_less(mpdu_seq_num, head_seq_num)) {
dev_kfree_skb(skb);
stats->last_rate = sta_stats_encode_rate(status);
stats->fragments++;
+ stats->packets++;
if (!(status->flag & RX_FLAG_NO_SIGNAL_VAL)) {
stats->last_signal = status->signal;
ieee80211_is_beacon(hdr->frame_control)))
ieee80211_scan_rx(local, skb);
- if (pubsta) {
- rx.sta = container_of(pubsta, struct sta_info, sta);
- rx.sdata = rx.sta->sdata;
- if (ieee80211_prepare_and_rx_handle(&rx, skb, true))
- return;
- goto out;
- } else if (ieee80211_is_data(fc)) {
+ if (ieee80211_is_data(fc)) {
struct sta_info *sta, *prev_sta;
+ if (pubsta) {
+ rx.sta = container_of(pubsta, struct sta_info, sta);
+ rx.sdata = rx.sta->sdata;
+ if (ieee80211_prepare_and_rx_handle(&rx, skb, true))
+ return;
+ goto out;
+ }
+
prev_sta = NULL;
for_each_sta_info(local, hdr->addr2, sta, tmp) {
}
/* No need to do anything if the driver does all */
- if (ieee80211_hw_check(&local->hw, AP_LINK_PS))
+ if (ieee80211_hw_check(&local->hw, AP_LINK_PS) && !local->ops->set_tim)
return;
if (sta->dead)
sta_info_recalc_tim(sta);
ps_dbg(sdata,
- "STA %pM aid %d sending %d filtered/%d PS frames since STA not sleeping anymore\n",
+ "STA %pM aid %d sending %d filtered/%d PS frames since STA woke up\n",
sta->sta.addr, sta->sta.aid, filtered, buffered);
ieee80211_check_fast_xmit(sta);
* @auto_seq: used for offloaded BA sessions to automatically pick head_seq_and
* and ssn.
* @removed: this session is removed (but might have been found due to RCU)
+ * @started: this session has started (head ssn or higher was received)
*
* This structure's lifetime is managed by RCU, assignments to
* the array holding it must hold the aggregation mutex.
u16 ssn;
u16 buf_size;
u16 timeout;
- bool auto_seq;
- bool removed;
+ u8 auto_seq:1,
+ removed:1,
+ started:1;
};
/**
unsigned int fail_avg;
};
-DECLARE_EWMA(signal, 1024, 8)
+DECLARE_EWMA(signal, 10, 8)
struct ieee80211_sta_rx_stats {
unsigned long packets;
struct ieee80211_hdr *hdr = (void *)skb->data;
int ac;
- if (info->flags & IEEE80211_TX_CTL_NO_PS_BUFFER) {
+ if (info->flags & (IEEE80211_TX_CTL_NO_PS_BUFFER |
+ IEEE80211_TX_CTL_AMPDU)) {
ieee80211_free_txskb(&local->hw, skb);
return;
}
#include <linux/bug.h>
#include <linux/completion.h>
#include <linux/ieee802154.h>
+#include <linux/rculist.h>
+
#include <crypto/aead.h>
#include <crypto/skcipher.h>
ports[ports_c++] = SIP_PORT;
for (i = 0; i < ports_c; i++) {
- memset(&sip[i], 0, sizeof(sip[i]));
-
nf_ct_helper_init(&sip[4 * i], AF_INET, IPPROTO_UDP, "sip",
SIP_PORT, ports[i], i, sip_exp_policy,
SIP_EXPECT_MAX,
return -1;
}
-static int nf_tables_table_notify(const struct nft_ctx *ctx, int event)
+static void nf_tables_table_notify(const struct nft_ctx *ctx, int event)
{
struct sk_buff *skb;
int err;
if (!ctx->report &&
!nfnetlink_has_listeners(ctx->net, NFNLGRP_NFTABLES))
- return 0;
+ return;
- err = -ENOBUFS;
skb = nlmsg_new(NLMSG_GOODSIZE, GFP_KERNEL);
if (skb == NULL)
goto err;
goto err;
}
- err = nfnetlink_send(skb, ctx->net, ctx->portid, NFNLGRP_NFTABLES,
- ctx->report, GFP_KERNEL);
+ nfnetlink_send(skb, ctx->net, ctx->portid, NFNLGRP_NFTABLES,
+ ctx->report, GFP_KERNEL);
+ return;
err:
- if (err < 0) {
- nfnetlink_set_err(ctx->net, ctx->portid, NFNLGRP_NFTABLES,
- err);
- }
- return err;
+ nfnetlink_set_err(ctx->net, ctx->portid, NFNLGRP_NFTABLES, -ENOBUFS);
}
static int nf_tables_dump_tables(struct sk_buff *skb,
return -1;
}
-static int nf_tables_chain_notify(const struct nft_ctx *ctx, int event)
+static void nf_tables_chain_notify(const struct nft_ctx *ctx, int event)
{
struct sk_buff *skb;
int err;
if (!ctx->report &&
!nfnetlink_has_listeners(ctx->net, NFNLGRP_NFTABLES))
- return 0;
+ return;
- err = -ENOBUFS;
skb = nlmsg_new(NLMSG_GOODSIZE, GFP_KERNEL);
if (skb == NULL)
goto err;
goto err;
}
- err = nfnetlink_send(skb, ctx->net, ctx->portid, NFNLGRP_NFTABLES,
- ctx->report, GFP_KERNEL);
+ nfnetlink_send(skb, ctx->net, ctx->portid, NFNLGRP_NFTABLES,
+ ctx->report, GFP_KERNEL);
+ return;
err:
- if (err < 0) {
- nfnetlink_set_err(ctx->net, ctx->portid, NFNLGRP_NFTABLES,
- err);
- }
- return err;
+ nfnetlink_set_err(ctx->net, ctx->portid, NFNLGRP_NFTABLES, -ENOBUFS);
}
static int nf_tables_dump_chains(struct sk_buff *skb,
return -1;
}
-static int nf_tables_rule_notify(const struct nft_ctx *ctx,
- const struct nft_rule *rule,
- int event)
+static void nf_tables_rule_notify(const struct nft_ctx *ctx,
+ const struct nft_rule *rule, int event)
{
struct sk_buff *skb;
int err;
if (!ctx->report &&
!nfnetlink_has_listeners(ctx->net, NFNLGRP_NFTABLES))
- return 0;
+ return;
- err = -ENOBUFS;
skb = nlmsg_new(NLMSG_GOODSIZE, GFP_KERNEL);
if (skb == NULL)
goto err;
goto err;
}
- err = nfnetlink_send(skb, ctx->net, ctx->portid, NFNLGRP_NFTABLES,
- ctx->report, GFP_KERNEL);
+ nfnetlink_send(skb, ctx->net, ctx->portid, NFNLGRP_NFTABLES,
+ ctx->report, GFP_KERNEL);
+ return;
err:
- if (err < 0) {
- nfnetlink_set_err(ctx->net, ctx->portid, NFNLGRP_NFTABLES,
- err);
- }
- return err;
+ nfnetlink_set_err(ctx->net, ctx->portid, NFNLGRP_NFTABLES, -ENOBUFS);
}
struct nft_rule_dump_ctx {
return -1;
}
-static int nf_tables_set_notify(const struct nft_ctx *ctx,
- const struct nft_set *set,
- int event, gfp_t gfp_flags)
+static void nf_tables_set_notify(const struct nft_ctx *ctx,
+ const struct nft_set *set, int event,
+ gfp_t gfp_flags)
{
struct sk_buff *skb;
u32 portid = ctx->portid;
if (!ctx->report &&
!nfnetlink_has_listeners(ctx->net, NFNLGRP_NFTABLES))
- return 0;
+ return;
- err = -ENOBUFS;
skb = nlmsg_new(NLMSG_GOODSIZE, gfp_flags);
if (skb == NULL)
goto err;
goto err;
}
- err = nfnetlink_send(skb, ctx->net, portid, NFNLGRP_NFTABLES,
- ctx->report, gfp_flags);
+ nfnetlink_send(skb, ctx->net, portid, NFNLGRP_NFTABLES, ctx->report,
+ gfp_flags);
+ return;
err:
- if (err < 0)
- nfnetlink_set_err(ctx->net, portid, NFNLGRP_NFTABLES, err);
- return err;
+ nfnetlink_set_err(ctx->net, portid, NFNLGRP_NFTABLES, -ENOBUFS);
}
static int nf_tables_dump_sets(struct sk_buff *skb, struct netlink_callback *cb)
return -1;
}
-static int nf_tables_setelem_notify(const struct nft_ctx *ctx,
- const struct nft_set *set,
- const struct nft_set_elem *elem,
- int event, u16 flags)
+static void nf_tables_setelem_notify(const struct nft_ctx *ctx,
+ const struct nft_set *set,
+ const struct nft_set_elem *elem,
+ int event, u16 flags)
{
struct net *net = ctx->net;
u32 portid = ctx->portid;
int err;
if (!ctx->report && !nfnetlink_has_listeners(net, NFNLGRP_NFTABLES))
- return 0;
+ return;
- err = -ENOBUFS;
skb = nlmsg_new(NLMSG_GOODSIZE, GFP_KERNEL);
if (skb == NULL)
goto err;
goto err;
}
- err = nfnetlink_send(skb, net, portid, NFNLGRP_NFTABLES, ctx->report,
- GFP_KERNEL);
+ nfnetlink_send(skb, net, portid, NFNLGRP_NFTABLES, ctx->report,
+ GFP_KERNEL);
+ return;
err:
- if (err < 0)
- nfnetlink_set_err(net, portid, NFNLGRP_NFTABLES, err);
- return err;
+ nfnetlink_set_err(net, portid, NFNLGRP_NFTABLES, -ENOBUFS);
}
static struct nft_trans *nft_trans_elem_alloc(struct nft_ctx *ctx,
return nft_delobj(&ctx, obj);
}
-int nft_obj_notify(struct net *net, struct nft_table *table,
- struct nft_object *obj, u32 portid, u32 seq, int event,
- int family, int report, gfp_t gfp)
+void nft_obj_notify(struct net *net, struct nft_table *table,
+ struct nft_object *obj, u32 portid, u32 seq, int event,
+ int family, int report, gfp_t gfp)
{
struct sk_buff *skb;
int err;
if (!report &&
!nfnetlink_has_listeners(net, NFNLGRP_NFTABLES))
- return 0;
+ return;
- err = -ENOBUFS;
skb = nlmsg_new(NLMSG_GOODSIZE, gfp);
if (skb == NULL)
goto err;
goto err;
}
- err = nfnetlink_send(skb, net, portid, NFNLGRP_NFTABLES, report, gfp);
+ nfnetlink_send(skb, net, portid, NFNLGRP_NFTABLES, report, gfp);
+ return;
err:
- if (err < 0) {
- nfnetlink_set_err(net, portid, NFNLGRP_NFTABLES, err);
- }
- return err;
+ nfnetlink_set_err(net, portid, NFNLGRP_NFTABLES, -ENOBUFS);
}
EXPORT_SYMBOL_GPL(nft_obj_notify);
-static int nf_tables_obj_notify(const struct nft_ctx *ctx,
- struct nft_object *obj, int event)
+static void nf_tables_obj_notify(const struct nft_ctx *ctx,
+ struct nft_object *obj, int event)
{
- return nft_obj_notify(ctx->net, ctx->table, obj, ctx->portid,
- ctx->seq, event, ctx->afi->family, ctx->report,
- GFP_KERNEL);
+ nft_obj_notify(ctx->net, ctx->table, obj, ctx->portid, ctx->seq, event,
+ ctx->afi->family, ctx->report, GFP_KERNEL);
}
static int nf_tables_fill_gen_info(struct sk_buff *skb, struct net *net,
return -EMSGSIZE;
}
-static int nf_tables_gen_notify(struct net *net, struct sk_buff *skb, int event)
+static void nf_tables_gen_notify(struct net *net, struct sk_buff *skb,
+ int event)
{
struct nlmsghdr *nlh = nlmsg_hdr(skb);
struct sk_buff *skb2;
if (nlmsg_report(nlh) &&
!nfnetlink_has_listeners(net, NFNLGRP_NFTABLES))
- return 0;
+ return;
- err = -ENOBUFS;
skb2 = nlmsg_new(NLMSG_GOODSIZE, GFP_KERNEL);
if (skb2 == NULL)
goto err;
goto err;
}
- err = nfnetlink_send(skb2, net, NETLINK_CB(skb).portid,
- NFNLGRP_NFTABLES, nlmsg_report(nlh), GFP_KERNEL);
+ nfnetlink_send(skb2, net, NETLINK_CB(skb).portid, NFNLGRP_NFTABLES,
+ nlmsg_report(nlh), GFP_KERNEL);
+ return;
err:
- if (err < 0) {
- nfnetlink_set_err(net, NETLINK_CB(skb).portid, NFNLGRP_NFTABLES,
- err);
- }
- return err;
+ nfnetlink_set_err(net, NETLINK_CB(skb).portid, NFNLGRP_NFTABLES,
+ -ENOBUFS);
}
static int nf_tables_getgen(struct net *net, struct sock *nlsk,
d = memcmp(this, key, set->klen);
if (d < 0) {
parent = parent->rb_left;
- /* In case of adjacent ranges, we always see the high
- * part of the range in first place, before the low one.
- * So don't update interval if the keys are equal.
- */
- if (interval && nft_rbtree_equal(set, this, interval))
+ if (interval &&
+ nft_rbtree_equal(set, this, interval) &&
+ nft_rbtree_interval_end(this) &&
+ !nft_rbtree_interval_end(interval))
continue;
interval = rbe;
} else if (d > 0)
#include <linux/module.h>
#include <linux/skbuff.h>
#include <linux/file.h>
+#include <linux/cred.h>
+
#include <net/sock.h>
#include <net/inet_sock.h>
#include <linux/netfilter/x_tables.h>
#include <linux/in.h>
#include <linux/slab.h>
#include <linux/kernel.h>
-#include <linux/sched.h>
+#include <linux/sched/signal.h>
#include <linux/timer.h>
#include <linux/string.h>
#include <linux/sockios.h>
#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/nfc.h>
+#include <linux/sched/signal.h>
#include "nfc.h"
#include "llcp.h"
unsigned long orig_dst;
struct rt6_info ovs_rt;
- if (!v6ops) {
+ if (!v6ops)
goto err;
- }
prepare_frag(vport, skb, orig_network_offset,
ovs_key_mac_proto(key));
} else if (key->eth.type == htons(ETH_P_IPV6)) {
enum ip6_defrag_users user = IP6_DEFRAG_CONNTRACK_IN + zone;
- skb_orphan(skb);
memset(IP6CB(skb), 0, sizeof(struct inet6_skb_parm));
err = nf_ct_frag6_gather(net, skb, user);
if (err) {
int addr_len)
{
struct sock *sk = sock->sk;
- char name[15];
+ char name[sizeof(uaddr->sa_data) + 1];
/*
* Check legality
if (addr_len != sizeof(struct sockaddr))
return -EINVAL;
- strlcpy(name, uaddr->sa_data, sizeof(name));
+ /* uaddr->sa_data comes from the userspace, it's not guaranteed to be
+ * zero-terminated.
+ */
+ memcpy(name, uaddr->sa_data, sizeof(uaddr->sa_data));
+ name[sizeof(uaddr->sa_data)] = 0;
return packet_do_bind(sk, name, 0, pkt_sk(sk)->num);
}
*/
#include <linux/kernel.h>
+#include <linux/sched/signal.h>
#include <linux/slab.h>
#include <linux/socket.h>
#include <net/sock.h>
#include <linux/kernel.h>
#include <linux/net.h>
#include <linux/poll.h>
+#include <linux/sched/signal.h>
+
#include <net/sock.h>
#include <net/tcp_states.h>
#include "ib.h"
#include "ib_mr.h"
-unsigned int rds_ib_mr_1m_pool_size = RDS_MR_1M_POOL_SIZE;
-unsigned int rds_ib_mr_8k_pool_size = RDS_MR_8K_POOL_SIZE;
+static unsigned int rds_ib_mr_1m_pool_size = RDS_MR_1M_POOL_SIZE;
+static unsigned int rds_ib_mr_8k_pool_size = RDS_MR_8K_POOL_SIZE;
unsigned int rds_ib_retry_count = RDS_IB_DEFAULT_RETRY_COUNT;
module_param(rds_ib_mr_1m_pool_size, int, 0444);
if (ret)
goto out_sysctl;
- ret = rds_trans_register(&rds_ib_transport);
- if (ret)
- goto out_recv;
+ rds_trans_register(&rds_ib_transport);
rds_info_register_func(RDS_INFO_IB_CONNECTIONS, rds_ib_ic_info);
goto out;
-out_recv:
- rds_ib_recv_exit();
out_sysctl:
rds_ib_sysctl_exit();
out_ibreg:
};
extern struct workqueue_struct *rds_ib_mr_wq;
-extern unsigned int rds_ib_mr_1m_pool_size;
-extern unsigned int rds_ib_mr_8k_pool_size;
extern bool prefer_frmr;
struct rds_ib_mr_pool *rds_ib_create_mr_pool(struct rds_ib_device *rds_dev,
static
DEFINE_PER_CPU_SHARED_ALIGNED(struct rds_page_remainder, rds_page_remainders);
-/*
- * returns 0 on success or -errno on failure.
- *
- * We don't have to worry about flush_dcache_page() as this only works
- * with private pages. If, say, we were to do directed receive to pinned
- * user pages we'd have to worry more about cache coherence. (Though
- * the flush_dcache_page() in get_user_pages() would probably be enough).
- */
-int rds_page_copy_user(struct page *page, unsigned long offset,
- void __user *ptr, unsigned long bytes,
- int to_user)
-{
- unsigned long ret;
- void *addr;
-
- addr = kmap(page);
- if (to_user) {
- rds_stats_add(s_copy_to_user, bytes);
- ret = copy_to_user(ptr, addr + offset, bytes);
- } else {
- rds_stats_add(s_copy_from_user, bytes);
- ret = copy_from_user(addr + offset, ptr, bytes);
- }
- kunmap(page);
-
- return ret ? -EFAULT : 0;
-}
-EXPORT_SYMBOL_GPL(rds_page_copy_user);
-
/**
* rds_page_remainder_alloc - build up regions of a message.
*
/* page.c */
int rds_page_remainder_alloc(struct scatterlist *scat, unsigned long bytes,
gfp_t gfp);
-int rds_page_copy_user(struct page *page, unsigned long offset,
- void __user *ptr, unsigned long bytes,
- int to_user);
-#define rds_page_copy_to_user(page, offset, ptr, bytes) \
- rds_page_copy_user(page, offset, ptr, bytes, 1)
-#define rds_page_copy_from_user(page, offset, ptr, bytes) \
- rds_page_copy_user(page, offset, ptr, bytes, 0)
void rds_page_exit(void);
/* recv.c */
void rds_connect_complete(struct rds_connection *conn);
/* transport.c */
-int rds_trans_register(struct rds_transport *trans);
+void rds_trans_register(struct rds_transport *trans);
void rds_trans_unregister(struct rds_transport *trans);
struct rds_transport *rds_trans_get_preferred(struct net *net, __be32 addr);
void rds_trans_put(struct rds_transport *trans);
if (ret)
goto out_pernet;
- ret = rds_trans_register(&rds_tcp_transport);
- if (ret)
- goto out_recv;
+ rds_trans_register(&rds_tcp_transport);
rds_info_register_func(RDS_INFO_TCP_SOCKETS, rds_tcp_tc_info);
goto out;
-out_recv:
- rds_tcp_recv_exit();
out_pernet:
unregister_pernet_subsys(&rds_tcp_net_ops);
out_notifier:
static struct rds_transport *transports[RDS_TRANS_COUNT];
static DECLARE_RWSEM(rds_trans_sem);
-int rds_trans_register(struct rds_transport *trans)
+void rds_trans_register(struct rds_transport *trans)
{
BUG_ON(strlen(trans->t_name) + 1 > TRANSNAMSIZ);
}
up_write(&rds_trans_sem);
-
- return 0;
}
EXPORT_SYMBOL_GPL(rds_trans_register);
#include <linux/in.h>
#include <linux/slab.h>
#include <linux/kernel.h>
-#include <linux/sched.h>
+#include <linux/sched/signal.h>
#include <linux/spinlock.h>
#include <linux/timer.h>
#include <linux/string.h>
cp.exclusive = false;
cp.service_id = srx->srx_service;
call = rxrpc_new_client_call(rx, &cp, srx, user_call_ID, gfp);
+ /* The socket has been unlocked. */
if (!IS_ERR(call))
call->notify_rx = notify_rx;
- release_sock(&rx->sk);
+ mutex_unlock(&call->user_mutex);
_leave(" = %p", call);
return call;
}
void rxrpc_kernel_end_call(struct socket *sock, struct rxrpc_call *call)
{
_enter("%d{%d}", call->debug_id, atomic_read(&call->usage));
+
+ mutex_lock(&call->user_mutex);
rxrpc_release_call(rxrpc_sk(sock->sk), call);
+ mutex_unlock(&call->user_mutex);
rxrpc_put_call(call, rxrpc_call_put_kernel);
}
EXPORT_SYMBOL(rxrpc_kernel_end_call);
case RXRPC_SERVER_BOUND:
case RXRPC_SERVER_LISTENING:
ret = rxrpc_do_sendmsg(rx, m, len);
- break;
+ /* The socket has been unlocked */
+ goto out;
default:
ret = -EINVAL;
- break;
+ goto error_unlock;
}
error_unlock:
release_sock(&rx->sk);
+out:
_leave(" = %d", ret);
return ret;
}
struct rxrpc_connection *conn; /* connection carrying call */
struct rxrpc_peer *peer; /* Peer record for remote address */
struct rxrpc_sock __rcu *socket; /* socket responsible */
+ struct mutex user_mutex; /* User access mutex */
ktime_t ack_at; /* When deferred ACK needs to happen */
ktime_t resend_at; /* When next resend needs to happen */
ktime_t ping_at; /* When next to send a ping */
*
* If we want to report an error, we mark the skb with the packet type and
* abort code and return NULL.
+ *
+ * The call is returned with the user access mutex held.
*/
struct rxrpc_call *rxrpc_new_incoming_call(struct rxrpc_local *local,
struct rxrpc_connection *conn,
trace_rxrpc_receive(call, rxrpc_receive_incoming,
sp->hdr.serial, sp->hdr.seq);
+ /* Lock the call to prevent rxrpc_kernel_send/recv_data() and
+ * sendmsg()/recvmsg() inconveniently stealing the mutex once the
+ * notification is generated.
+ *
+ * The BUG should never happen because the kernel should be well
+ * behaved enough not to access the call before the first notification
+ * event and userspace is prevented from doing so until the state is
+ * appropriate.
+ */
+ if (!mutex_trylock(&call->user_mutex))
+ BUG();
+
/* Make the call live. */
rxrpc_incoming_call(rx, call, skb);
conn = call->conn;
/*
* handle acceptance of a call by userspace
* - assign the user call ID to the call at the front of the queue
+ * - called with the socket locked.
*/
struct rxrpc_call *rxrpc_accept_call(struct rxrpc_sock *rx,
unsigned long user_call_ID,
rxrpc_notify_rx_t notify_rx)
+ __releases(&rx->sk.sk_lock.slock)
{
struct rxrpc_call *call;
struct rb_node *parent, **pp;
if (list_empty(&rx->to_be_accepted)) {
write_unlock(&rx->call_lock);
+ release_sock(&rx->sk);
kleave(" = -ENODATA [empty]");
return ERR_PTR(-ENODATA);
}
*/
call = list_entry(rx->to_be_accepted.next,
struct rxrpc_call, accept_link);
+ write_unlock(&rx->call_lock);
+
+ /* We need to gain the mutex from the interrupt handler without
+ * upsetting lockdep, so we have to release it there and take it here.
+ * We are, however, still holding the socket lock, so other accepts
+ * must wait for us and no one can add the user ID behind our backs.
+ */
+ if (mutex_lock_interruptible(&call->user_mutex) < 0) {
+ release_sock(&rx->sk);
+ kleave(" = -ERESTARTSYS");
+ return ERR_PTR(-ERESTARTSYS);
+ }
+
+ write_lock(&rx->call_lock);
list_del_init(&call->accept_link);
sk_acceptq_removed(&rx->sk);
rxrpc_see_call(call);
+ /* Find the user ID insertion point. */
+ pp = &rx->calls.rb_node;
+ parent = NULL;
+ while (*pp) {
+ parent = *pp;
+ call = rb_entry(parent, struct rxrpc_call, sock_node);
+
+ if (user_call_ID < call->user_call_ID)
+ pp = &(*pp)->rb_left;
+ else if (user_call_ID > call->user_call_ID)
+ pp = &(*pp)->rb_right;
+ else
+ BUG();
+ }
+
write_lock_bh(&call->state_lock);
switch (call->state) {
case RXRPC_CALL_SERVER_ACCEPTING:
write_unlock(&rx->call_lock);
rxrpc_notify_socket(call);
rxrpc_service_prealloc(rx, GFP_KERNEL);
+ release_sock(&rx->sk);
_leave(" = %p{%d}", call, call->debug_id);
return call;
write_unlock(&rx->call_lock);
out:
rxrpc_service_prealloc(rx, GFP_KERNEL);
+ release_sock(&rx->sk);
_leave(" = %d", ret);
return ERR_PTR(ret);
}
if (!call->rxtx_annotations)
goto nomem_2;
+ mutex_init(&call->user_mutex);
setup_timer(&call->timer, rxrpc_call_timer_expired,
(unsigned long)call);
INIT_WORK(&call->processor, &rxrpc_process_call);
}
/*
- * set up a call for the given data
- * - called in process context with IRQs enabled
+ * Set up a call for the given parameters.
+ * - Called with the socket lock held, which it must release.
+ * - If it returns a call, the call's lock will need releasing by the caller.
*/
struct rxrpc_call *rxrpc_new_client_call(struct rxrpc_sock *rx,
struct rxrpc_conn_parameters *cp,
struct sockaddr_rxrpc *srx,
unsigned long user_call_ID,
gfp_t gfp)
+ __releases(&rx->sk.sk_lock.slock)
{
struct rxrpc_call *call, *xcall;
struct rb_node *parent, **pp;
call = rxrpc_alloc_client_call(srx, gfp);
if (IS_ERR(call)) {
+ release_sock(&rx->sk);
_leave(" = %ld", PTR_ERR(call));
return call;
}
trace_rxrpc_call(call, rxrpc_call_new_client, atomic_read(&call->usage),
here, (const void *)user_call_ID);
+ /* We need to protect a partially set up call against the user as we
+ * will be acting outside the socket lock.
+ */
+ mutex_lock(&call->user_mutex);
+
/* Publish the call, even though it is incompletely set up as yet */
write_lock(&rx->call_lock);
list_add_tail(&call->link, &rxrpc_calls);
write_unlock(&rxrpc_call_lock);
+ /* From this point on, the call is protected by its own lock. */
+ release_sock(&rx->sk);
+
/* Set up or get a connection record and set the protocol parameters,
* including channel number and call ID.
*/
*/
error_dup_user_ID:
write_unlock(&rx->call_lock);
+ release_sock(&rx->sk);
ret = -EEXIST;
error:
trace_rxrpc_call(call, rxrpc_call_error, atomic_read(&call->usage),
here, ERR_PTR(ret));
rxrpc_release_call(rx, call);
+ mutex_unlock(&call->user_mutex);
rxrpc_put_call(call, rxrpc_call_put);
_leave(" = %d", ret);
return ERR_PTR(ret);
#include <linux/slab.h>
#include <linux/idr.h>
#include <linux/timer.h>
+#include <linux/sched/signal.h>
+
#include "ar-internal.h"
__read_mostly unsigned int rxrpc_max_client_connections = 1000;
goto reject_packet;
}
rxrpc_send_ping(call, skb, skew);
+ mutex_unlock(&call->user_mutex);
}
rxrpc_input_call_packet(call, skb, skew);
#include <linux/net.h>
#include <linux/skbuff.h>
#include <linux/export.h>
+#include <linux/sched/signal.h>
+
#include <net/sock.h>
#include <net/af_rxrpc.h>
#include "ar-internal.h"
trace_rxrpc_recvmsg(call, rxrpc_recvmsg_dequeue, 0, 0, 0, 0);
+ /* We're going to drop the socket lock, so we need to lock the call
+ * against interference by sendmsg.
+ */
+ if (!mutex_trylock(&call->user_mutex)) {
+ ret = -EWOULDBLOCK;
+ if (flags & MSG_DONTWAIT)
+ goto error_requeue_call;
+ ret = -ERESTARTSYS;
+ if (mutex_lock_interruptible(&call->user_mutex) < 0)
+ goto error_requeue_call;
+ }
+
+ release_sock(&rx->sk);
+
if (test_bit(RXRPC_CALL_RELEASED, &call->flags))
BUG();
&call->user_call_ID);
}
if (ret < 0)
- goto error;
+ goto error_unlock_call;
}
if (msg->msg_name) {
}
if (ret < 0)
- goto error;
+ goto error_unlock_call;
if (call->state == RXRPC_CALL_COMPLETE) {
ret = rxrpc_recvmsg_term(call, msg);
if (ret < 0)
- goto error;
+ goto error_unlock_call;
if (!(flags & MSG_PEEK))
rxrpc_release_call(rx, call);
msg->msg_flags |= MSG_EOR;
msg->msg_flags &= ~MSG_MORE;
ret = copied;
-error:
+error_unlock_call:
+ mutex_unlock(&call->user_mutex);
rxrpc_put_call(call, rxrpc_call_put);
+ trace_rxrpc_recvmsg(call, rxrpc_recvmsg_return, 0, 0, 0, ret);
+ return ret;
+
+error_requeue_call:
+ if (!(flags & MSG_PEEK)) {
+ write_lock_bh(&rx->recvmsg_lock);
+ list_add(&call->recvmsg_link, &rx->recvmsg_q);
+ write_unlock_bh(&rx->recvmsg_lock);
+ trace_rxrpc_recvmsg(call, rxrpc_recvmsg_requeue, 0, 0, 0, 0);
+ } else {
+ rxrpc_put_call(call, rxrpc_call_put);
+ }
error_no_call:
release_sock(&rx->sk);
trace_rxrpc_recvmsg(call, rxrpc_recvmsg_return, 0, 0, 0, ret);
iov.iov_len = size - *_offset;
iov_iter_kvec(&iter, ITER_KVEC | READ, &iov, 1, size - *_offset);
- lock_sock(sock->sk);
+ mutex_lock(&call->user_mutex);
switch (call->state) {
case RXRPC_CALL_CLIENT_RECV_REPLY:
read_phase_complete:
ret = 1;
out:
- release_sock(sock->sk);
+ mutex_unlock(&call->user_mutex);
_leave(" = %d [%zu,%d]", ret, *_offset, *_abort);
return ret;
#include <linux/gfp.h>
#include <linux/skbuff.h>
#include <linux/export.h>
+#include <linux/sched/signal.h>
+
#include <net/sock.h>
#include <net/af_rxrpc.h>
#include "ar-internal.h"
}
trace_rxrpc_transmit(call, rxrpc_transmit_wait);
- release_sock(&rx->sk);
+ mutex_unlock(&call->user_mutex);
*timeo = schedule_timeout(*timeo);
- lock_sock(&rx->sk);
+ if (mutex_lock_interruptible(&call->user_mutex) < 0) {
+ ret = sock_intr_errno(*timeo);
+ break;
+ }
}
remove_wait_queue(&call->waitq, &myself);
/*
* send data through a socket
* - must be called in process context
- * - caller holds the socket locked
+ * - The caller holds the call user access mutex, but not the socket lock.
*/
static int rxrpc_send_data(struct rxrpc_sock *rx,
struct rxrpc_call *call,
/*
* Create a new client call for sendmsg().
+ * - Called with the socket lock held, which it must release.
+ * - If it returns a call, the call's lock will need releasing by the caller.
*/
static struct rxrpc_call *
rxrpc_new_client_call_for_sendmsg(struct rxrpc_sock *rx, struct msghdr *msg,
unsigned long user_call_ID, bool exclusive)
+ __releases(&rx->sk.sk_lock.slock)
{
struct rxrpc_conn_parameters cp;
struct rxrpc_call *call;
_enter("");
- if (!msg->msg_name)
+ if (!msg->msg_name) {
+ release_sock(&rx->sk);
return ERR_PTR(-EDESTADDRREQ);
+ }
key = rx->key;
if (key && !rx->key->payload.data[0])
cp.exclusive = rx->exclusive | exclusive;
cp.service_id = srx->srx_service;
call = rxrpc_new_client_call(rx, &cp, srx, user_call_ID, GFP_KERNEL);
+ /* The socket is now unlocked */
_leave(" = %p\n", call);
return call;
* - the socket may be either a client socket or a server socket
*/
int rxrpc_do_sendmsg(struct rxrpc_sock *rx, struct msghdr *msg, size_t len)
+ __releases(&rx->sk.sk_lock.slock)
{
enum rxrpc_command cmd;
struct rxrpc_call *call;
ret = rxrpc_sendmsg_cmsg(msg, &user_call_ID, &cmd, &abort_code,
&exclusive);
if (ret < 0)
- return ret;
+ goto error_release_sock;
if (cmd == RXRPC_CMD_ACCEPT) {
+ ret = -EINVAL;
if (rx->sk.sk_state != RXRPC_SERVER_LISTENING)
- return -EINVAL;
+ goto error_release_sock;
call = rxrpc_accept_call(rx, user_call_ID, NULL);
+ /* The socket is now unlocked. */
if (IS_ERR(call))
return PTR_ERR(call);
rxrpc_put_call(call, rxrpc_call_put);
call = rxrpc_find_call_by_user_ID(rx, user_call_ID);
if (!call) {
+ ret = -EBADSLT;
if (cmd != RXRPC_CMD_SEND_DATA)
- return -EBADSLT;
+ goto error_release_sock;
call = rxrpc_new_client_call_for_sendmsg(rx, msg, user_call_ID,
exclusive);
+ /* The socket is now unlocked... */
if (IS_ERR(call))
return PTR_ERR(call);
+ /* ... and we have the call lock. */
+ } else {
+ ret = -EBUSY;
+ if (call->state == RXRPC_CALL_UNINITIALISED ||
+ call->state == RXRPC_CALL_CLIENT_AWAIT_CONN ||
+ call->state == RXRPC_CALL_SERVER_PREALLOC ||
+ call->state == RXRPC_CALL_SERVER_SECURING ||
+ call->state == RXRPC_CALL_SERVER_ACCEPTING)
+ goto error_release_sock;
+
+ ret = mutex_lock_interruptible(&call->user_mutex);
+ release_sock(&rx->sk);
+ if (ret < 0) {
+ ret = -ERESTARTSYS;
+ goto error_put;
+ }
}
_debug("CALL %d USR %lx ST %d on CONN %p",
ret = rxrpc_send_data(rx, call, msg, len);
}
+ mutex_unlock(&call->user_mutex);
+error_put:
rxrpc_put_call(call, rxrpc_call_put);
_leave(" = %d", ret);
return ret;
+
+error_release_sock:
+ release_sock(&rx->sk);
+ return ret;
}
/**
ASSERTCMP(msg->msg_name, ==, NULL);
ASSERTCMP(msg->msg_control, ==, NULL);
- lock_sock(sock->sk);
+ mutex_lock(&call->user_mutex);
_debug("CALL %d USR %lx ST %d on CONN %p",
call->debug_id, call->user_call_ID, call->state, call->conn);
ret = rxrpc_send_data(rxrpc_sk(sock->sk), call, msg, len);
}
- release_sock(sock->sk);
+ mutex_unlock(&call->user_mutex);
_leave(" = %d", ret);
return ret;
}
{
_enter("{%d},%d,%d,%s", call->debug_id, abort_code, error, why);
- lock_sock(sock->sk);
+ mutex_lock(&call->user_mutex);
if (rxrpc_abort_call(why, call, 0, abort_code, error))
rxrpc_send_abort_packet(call);
- release_sock(sock->sk);
+ mutex_unlock(&call->user_mutex);
_leave("");
}
#include <linux/types.h>
#include <linux/kernel.h>
#include <linux/sched.h>
+#include <linux/sched/loadavg.h>
#include <linux/string.h>
#include <linux/skbuff.h>
#include <linux/random.h>
arg.paddr = &t->ipaddr;
arg.lport = htons(t->asoc->base.bind_addr.port);
+ rcu_read_lock();
list = rhltable_lookup(&sctp_transport_hashtable, &arg,
sctp_hash_params);
rhl_for_each_entry_rcu(transport, tmp, list, node)
if (transport->asoc->ep == t->asoc->ep) {
+ rcu_read_unlock();
err = -EEXIST;
goto out;
}
+ rcu_read_unlock();
err = rhltable_insert_key(&sctp_transport_hashtable, &arg,
&t->node, sctp_hash_params);
#include <linux/kernel.h>
#include <linux/wait.h>
#include <linux/time.h>
+#include <linux/sched/signal.h>
#include <linux/ip.h>
#include <linux/capability.h>
#include <linux/fcntl.h>
#include <linux/inetdevice.h>
#include <linux/workqueue.h>
#include <linux/in.h>
+#include <linux/sched/signal.h>
+
#include <net/sock.h>
#include <net/tcp.h>
#include <net/smc.h>
#include <linux/in.h>
#include <linux/if_ether.h>
+#include <linux/sched/signal.h>
+
#include <net/sock.h>
#include <net/tcp.h>
*/
#include <linux/workqueue.h>
+#include <linux/sched/signal.h>
+
#include <net/sock.h>
#include "smc.h"
#include <linux/net.h>
#include <linux/rcupdate.h>
+#include <linux/sched/signal.h>
+
#include <net/sock.h>
#include "smc.h"
#include <linux/net.h>
#include <linux/rcupdate.h>
#include <linux/workqueue.h>
+#include <linux/sched/signal.h>
+
#include <net/sock.h>
#include "smc.h"
static void __exit strp_mod_exit(void)
{
+ destroy_workqueue(strp_wq);
}
module_init(strp_mod_init);
module_exit(strp_mod_exit);
#include <linux/types.h>
#include <linux/sched.h>
+#include <linux/cred.h>
#include <linux/module.h>
#include <linux/slab.h>
#include <linux/errno.h>
*/
#include <linux/linkage.h>
-#include <linux/sched.h>
+#include <linux/sched/signal.h>
#include <linux/errno.h>
#include <linux/net.h>
#include <linux/in.h>
*/
#include <linux/rhashtable.h>
+#include <linux/sched/signal.h>
+
#include "core.h"
#include "name_table.h"
#include "node.h"
#include <linux/module.h>
#include <linux/kernel.h>
#include <linux/signal.h>
-#include <linux/sched.h>
+#include <linux/sched/signal.h>
#include <linux/errno.h>
#include <linux/string.h>
#include <linux/stat.h>
#include <linux/init.h>
#include <linux/io.h>
#include <linux/kernel.h>
+#include <linux/sched/signal.h>
#include <linux/kmod.h>
#include <linux/list.h>
#include <linux/miscdevice.h>
vsock->vdev = vdev;
ret = vsock->vdev->config->find_vqs(vsock->vdev, VSOCK_VQ_MAX,
- vsock->vqs, callbacks, names);
+ vsock->vqs, callbacks, names,
+ NULL);
if (ret < 0)
goto out;
*/
#include <linux/spinlock.h>
#include <linux/module.h>
+#include <linux/sched/signal.h>
#include <linux/ctype.h>
#include <linux/list.h>
#include <linux/virtio.h>
#include <linux/capability.h>
#include <linux/errno.h>
#include <linux/kernel.h>
-#include <linux/sched.h>
+#include <linux/sched/signal.h>
#include <linux/timer.h>
#include <linux/string.h>
#include <linux/net.h>
Build a virtual tty sample driver for use as a VFIO
mediated device
+config SAMPLE_STATX
+ bool "Build example extended-stat using code"
+ depends on BROKEN
+ help
+ Build example userspace program to use the new extended-stat syscall.
+
endif # SAMPLES
obj-$(CONFIG_SAMPLES) += kobject/ kprobes/ trace_events/ livepatch/ \
hw_breakpoint/ kfifo/ kdb/ hidraw/ rpmsg/ seccomp/ \
configfs/ connector/ v4l/ trace_printk/ blackfin/ \
- vfio-mdev/
+ vfio-mdev/ statx/
--- /dev/null
+# kbuild trick to avoid linker error. Can be omitted if a module is built.
+obj- := dummy.o
+
+# List of programs to build
+hostprogs-$(CONFIG_SAMPLE_STATX) := test-statx
+
+# Tell kbuild to always build the programs
+always := $(hostprogs-y)
+
+HOSTCFLAGS_test-statx.o += -I$(objtree)/usr/include
--- /dev/null
+/* Test the statx() system call.
+ *
+ * Note that the output of this program is intended to look like the output of
+ * /bin/stat where possible.
+ *
+ * Copyright (C) 2015 Red Hat, Inc. All Rights Reserved.
+ * Written by David Howells (dhowells@redhat.com)
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public Licence
+ * as published by the Free Software Foundation; either version
+ * 2 of the Licence, or (at your option) any later version.
+ */
+
+#define _GNU_SOURCE
+#define _ATFILE_SOURCE
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+#include <ctype.h>
+#include <errno.h>
+#include <time.h>
+#include <sys/syscall.h>
+#include <sys/types.h>
+#include <linux/stat.h>
+#include <linux/fcntl.h>
+#include <sys/stat.h>
+
+#define AT_STATX_SYNC_TYPE 0x6000
+#define AT_STATX_SYNC_AS_STAT 0x0000
+#define AT_STATX_FORCE_SYNC 0x2000
+#define AT_STATX_DONT_SYNC 0x4000
+
+static __attribute__((unused))
+ssize_t statx(int dfd, const char *filename, unsigned flags,
+ unsigned int mask, struct statx *buffer)
+{
+ return syscall(__NR_statx, dfd, filename, flags, mask, buffer);
+}
+
+static void print_time(const char *field, struct statx_timestamp *ts)
+{
+ struct tm tm;
+ time_t tim;
+ char buffer[100];
+ int len;
+
+ tim = ts->tv_sec;
+ if (!localtime_r(&tim, &tm)) {
+ perror("localtime_r");
+ exit(1);
+ }
+ len = strftime(buffer, 100, "%F %T", &tm);
+ if (len == 0) {
+ perror("strftime");
+ exit(1);
+ }
+ printf("%s", field);
+ fwrite(buffer, 1, len, stdout);
+ printf(".%09u", ts->tv_nsec);
+ len = strftime(buffer, 100, "%z", &tm);
+ if (len == 0) {
+ perror("strftime2");
+ exit(1);
+ }
+ fwrite(buffer, 1, len, stdout);
+ printf("\n");
+}
+
+static void dump_statx(struct statx *stx)
+{
+ char buffer[256], ft = '?';
+
+ printf("results=%x\n", stx->stx_mask);
+
+ printf(" ");
+ if (stx->stx_mask & STATX_SIZE)
+ printf(" Size: %-15llu", (unsigned long long)stx->stx_size);
+ if (stx->stx_mask & STATX_BLOCKS)
+ printf(" Blocks: %-10llu", (unsigned long long)stx->stx_blocks);
+ printf(" IO Block: %-6llu", (unsigned long long)stx->stx_blksize);
+ if (stx->stx_mask & STATX_TYPE) {
+ switch (stx->stx_mode & S_IFMT) {
+ case S_IFIFO: printf(" FIFO\n"); ft = 'p'; break;
+ case S_IFCHR: printf(" character special file\n"); ft = 'c'; break;
+ case S_IFDIR: printf(" directory\n"); ft = 'd'; break;
+ case S_IFBLK: printf(" block special file\n"); ft = 'b'; break;
+ case S_IFREG: printf(" regular file\n"); ft = '-'; break;
+ case S_IFLNK: printf(" symbolic link\n"); ft = 'l'; break;
+ case S_IFSOCK: printf(" socket\n"); ft = 's'; break;
+ default:
+ printf(" unknown type (%o)\n", stx->stx_mode & S_IFMT);
+ break;
+ }
+ } else {
+ printf(" no type\n");
+ }
+
+ sprintf(buffer, "%02x:%02x", stx->stx_dev_major, stx->stx_dev_minor);
+ printf("Device: %-15s", buffer);
+ if (stx->stx_mask & STATX_INO)
+ printf(" Inode: %-11llu", (unsigned long long) stx->stx_ino);
+ if (stx->stx_mask & STATX_NLINK)
+ printf(" Links: %-5u", stx->stx_nlink);
+ if (stx->stx_mask & STATX_TYPE) {
+ switch (stx->stx_mode & S_IFMT) {
+ case S_IFBLK:
+ case S_IFCHR:
+ printf(" Device type: %u,%u",
+ stx->stx_rdev_major, stx->stx_rdev_minor);
+ break;
+ }
+ }
+ printf("\n");
+
+ if (stx->stx_mask & STATX_MODE)
+ printf("Access: (%04o/%c%c%c%c%c%c%c%c%c%c) ",
+ stx->stx_mode & 07777,
+ ft,
+ stx->stx_mode & S_IRUSR ? 'r' : '-',
+ stx->stx_mode & S_IWUSR ? 'w' : '-',
+ stx->stx_mode & S_IXUSR ? 'x' : '-',
+ stx->stx_mode & S_IRGRP ? 'r' : '-',
+ stx->stx_mode & S_IWGRP ? 'w' : '-',
+ stx->stx_mode & S_IXGRP ? 'x' : '-',
+ stx->stx_mode & S_IROTH ? 'r' : '-',
+ stx->stx_mode & S_IWOTH ? 'w' : '-',
+ stx->stx_mode & S_IXOTH ? 'x' : '-');
+ if (stx->stx_mask & STATX_UID)
+ printf("Uid: %5d ", stx->stx_uid);
+ if (stx->stx_mask & STATX_GID)
+ printf("Gid: %5d\n", stx->stx_gid);
+
+ if (stx->stx_mask & STATX_ATIME)
+ print_time("Access: ", &stx->stx_atime);
+ if (stx->stx_mask & STATX_MTIME)
+ print_time("Modify: ", &stx->stx_mtime);
+ if (stx->stx_mask & STATX_CTIME)
+ print_time("Change: ", &stx->stx_ctime);
+ if (stx->stx_mask & STATX_BTIME)
+ print_time(" Birth: ", &stx->stx_btime);
+
+ if (stx->stx_attributes) {
+ unsigned char bits;
+ int loop, byte;
+
+ static char attr_representation[64 + 1] =
+ /* STATX_ATTR_ flags: */
+ "????????" /* 63-56 */
+ "????????" /* 55-48 */
+ "????????" /* 47-40 */
+ "????????" /* 39-32 */
+ "????????" /* 31-24 0x00000000-ff000000 */
+ "????????" /* 23-16 0x00000000-00ff0000 */
+ "???me???" /* 15- 8 0x00000000-0000ff00 */
+ "?dai?c??" /* 7- 0 0x00000000-000000ff */
+ ;
+
+ printf("Attributes: %016llx (", stx->stx_attributes);
+ for (byte = 64 - 8; byte >= 0; byte -= 8) {
+ bits = stx->stx_attributes >> byte;
+ for (loop = 7; loop >= 0; loop--) {
+ int bit = byte + loop;
+
+ if (bits & 0x80)
+ putchar(attr_representation[63 - bit]);
+ else
+ putchar('-');
+ bits <<= 1;
+ }
+ if (byte)
+ putchar(' ');
+ }
+ printf(")\n");
+ }
+}
+
+static void dump_hex(unsigned long long *data, int from, int to)
+{
+ unsigned offset, print_offset = 1, col = 0;
+
+ from /= 8;
+ to = (to + 7) / 8;
+
+ for (offset = from; offset < to; offset++) {
+ if (print_offset) {
+ printf("%04x: ", offset * 8);
+ print_offset = 0;
+ }
+ printf("%016llx", data[offset]);
+ col++;
+ if ((col & 3) == 0) {
+ printf("\n");
+ print_offset = 1;
+ } else {
+ printf(" ");
+ }
+ }
+
+ if (!print_offset)
+ printf("\n");
+}
+
+int main(int argc, char **argv)
+{
+ struct statx stx;
+ int ret, raw = 0, atflag = AT_SYMLINK_NOFOLLOW;
+
+ unsigned int mask = STATX_ALL;
+
+ for (argv++; *argv; argv++) {
+ if (strcmp(*argv, "-F") == 0) {
+ atflag &= ~AT_STATX_SYNC_TYPE;
+ atflag |= AT_STATX_FORCE_SYNC;
+ continue;
+ }
+ if (strcmp(*argv, "-D") == 0) {
+ atflag &= ~AT_STATX_SYNC_TYPE;
+ atflag |= AT_STATX_DONT_SYNC;
+ continue;
+ }
+ if (strcmp(*argv, "-L") == 0) {
+ atflag &= ~AT_SYMLINK_NOFOLLOW;
+ continue;
+ }
+ if (strcmp(*argv, "-O") == 0) {
+ mask &= ~STATX_BASIC_STATS;
+ continue;
+ }
+ if (strcmp(*argv, "-A") == 0) {
+ atflag |= AT_NO_AUTOMOUNT;
+ continue;
+ }
+ if (strcmp(*argv, "-R") == 0) {
+ raw = 1;
+ continue;
+ }
+
+ memset(&stx, 0xbf, sizeof(stx));
+ ret = statx(AT_FDCWD, *argv, atflag, mask, &stx);
+ printf("statx(%s) = %d\n", *argv, ret);
+ if (ret < 0) {
+ perror(*argv);
+ exit(1);
+ }
+
+ if (raw)
+ dump_hex((unsigned long long *)&stx, 0, sizeof(stx));
+
+ dump_statx(&stx);
+ }
+ return 0;
+}
/* Silly tracepoints */
trace_foo_bar("hello", cnt, array, random_strings[len],
- tsk_cpus_allowed(current));
+ ¤t->cpus_allowed);
trace_foo_with_template_simple("HELLO", cnt);
#if BUILDING_GCC_VERSION < 6000
register_callback(plugin_name, PLUGIN_START_UNIT, &sancov_start_unit, NULL);
register_callback(plugin_name, PLUGIN_REGISTER_GGC_ROOTS, NULL, (void *)>_ggc_r_gt_sancov);
- register_callback(plugin_name, PLUGIN_PASS_MANAGER_SETUP, NULL, &sancov_plugin_pass_info);
+ register_callback(plugin_name, PLUGIN_PASS_MANAGER_SETUP, NULL, &sancov_pass_info);
#endif
return 0;
. = ALIGN(8);
.init_array 0 : { *(SORT(.init_array.*)) *(.init_array) }
+
+ __jump_table 0 : ALIGN(8) { KEEP(*(__jump_table)) }
}
disapear||disappear
disapeared||disappeared
disappared||disappeared
+disble||disable
+disbled||disabled
disconnet||disconnect
discontinous||discontinuous
dispertion||dispersion
overaall||overall
overhread||overhead
overlaping||overlapping
+overide||override
overrided||overridden
overriden||overridden
overun||overrun
#include <linux/slab.h>
#include <linux/spinlock.h>
#include <linux/string.h>
+#include <linux/cred.h>
+#include <linux/rculist.h>
#include <linux/user_namespace.h>
#include "include/apparmor.h"
#include <linux/xattr.h>
#include <linux/integrity.h>
#include <linux/evm.h>
+#include <linux/magic.h>
+
#include <crypto/hash.h>
#include <crypto/algapi.h>
#include "evm.h"
if (status == 0) {
const struct user_key_payload *payload;
- payload = user_key_payload(key);
+ payload = user_key_payload_locked(key);
if (maxlen == 0) {
*mpi = NULL;
goto error;
down_read(&ukey->sem);
- upayload = user_key_payload(ukey);
+ upayload = user_key_payload_locked(ukey);
*master_key = upayload->data;
*master_keylen = upayload->datalen;
error:
size_t asciiblob_len;
int ret;
- epayload = rcu_dereference_key(key);
+ epayload = dereference_key_locked(key);
/* returns the hex encoded iv, encrypted-data, and hmac as ascii */
asciiblob_len = epayload->datablob_len + ivsize + 1
#define _INTERNAL_H
#include <linux/sched.h>
+#include <linux/cred.h>
#include <linux/key-type.h>
#include <linux/task_work.h>
#include <linux/keyctl.h>
#include <linux/module.h>
#include <linux/init.h>
#include <linux/sched.h>
+#include <linux/sched/task.h>
#include <linux/slab.h>
#include <linux/syscalls.h>
#include <linux/key.h>
#include <linux/keyctl.h>
#include <linux/fs.h>
#include <linux/capability.h>
+#include <linux/cred.h>
#include <linux/string.h>
#include <linux/err.h>
#include <linux/vmalloc.h>
*/
#include <linux/user_namespace.h>
+#include <linux/cred.h>
+
#include "internal.h"
unsigned persistent_keyring_expiry = 3 * 24 * 3600; /* Expire after 3 days of non-use */
#include <linux/module.h>
#include <linux/init.h>
#include <linux/sched.h>
+#include <linux/sched/user.h>
#include <linux/keyctl.h>
#include <linux/fs.h>
#include <linux/err.h>
static long trusted_read(const struct key *key, char __user *buffer,
size_t buflen)
{
- struct trusted_key_payload *p;
+ const struct trusted_key_payload *p;
char *ascii_buf;
char *bufp;
int i;
- p = rcu_dereference_key(key);
+ p = dereference_key_locked(key);
if (!p)
return -EINVAL;
if (!buffer || buflen <= 0)
/* attach the new data, displacing the old */
key->expiry = prep->expiry;
if (!test_bit(KEY_FLAG_NEGATIVE, &key->flags))
- zap = rcu_dereference_key(key);
+ zap = dereference_key_locked(key);
rcu_assign_keypointer(key, prep->payload.data[0]);
prep->payload.data[0] = NULL;
*/
void user_revoke(struct key *key)
{
- struct user_key_payload *upayload = key->payload.data[0];
+ struct user_key_payload *upayload = user_key_payload_locked(key);
/* clear the quota */
key_payload_reserve(key, 0);
const struct user_key_payload *upayload;
long ret;
- upayload = user_key_payload(key);
+ upayload = user_key_payload_locked(key);
ret = upayload->datalen;
/* we can return the data as is */
#include <linux/kernel.h>
#include <linux/tracehook.h>
#include <linux/errno.h>
-#include <linux/sched.h>
+#include <linux/sched/signal.h>
+#include <linux/sched/task.h>
#include <linux/lsm_hooks.h>
#include <linux/xattr.h>
#include <linux/capability.h>
sbsec->behavior == SECURITY_FS_USE_NATIVE ||
/* Special handling. Genfs but also in-core setxattr handler */
!strcmp(sb->s_type->name, "sysfs") ||
- !strcmp(sb->s_type->name, "cgroup") ||
- !strcmp(sb->s_type->name, "cgroup2") ||
!strcmp(sb->s_type->name, "pstore") ||
!strcmp(sb->s_type->name, "debugfs") ||
!strcmp(sb->s_type->name, "tracefs") ||
- !strcmp(sb->s_type->name, "rootfs");
+ !strcmp(sb->s_type->name, "rootfs") ||
+ (selinux_policycap_cgroupseclabel &&
+ (!strcmp(sb->s_type->name, "cgroup") ||
+ !strcmp(sb->s_type->name, "cgroup2")));
}
static int sb_finish_set_opts(struct super_block *sb)
POLICYDB_CAPABILITY_OPENPERM,
POLICYDB_CAPABILITY_EXTSOCKCLASS,
POLICYDB_CAPABILITY_ALWAYSNETWORK,
+ POLICYDB_CAPABILITY_CGROUPSECLABEL,
__POLICYDB_CAPABILITY_MAX
};
#define POLICYDB_CAPABILITY_MAX (__POLICYDB_CAPABILITY_MAX - 1)
extern int selinux_policycap_openperm;
extern int selinux_policycap_extsockclass;
extern int selinux_policycap_alwaysnetwork;
+extern int selinux_policycap_cgroupseclabel;
/*
* type_datum properties
"network_peer_controls",
"open_perms",
"extended_socket_class",
- "always_check_network"
+ "always_check_network",
+ "cgroup_seclabel"
};
unsigned int selinux_checkreqprot = CONFIG_SECURITY_SELINUX_CHECKREQPROT_VALUE;
int selinux_policycap_openperm;
int selinux_policycap_extsockclass;
int selinux_policycap_alwaysnetwork;
+int selinux_policycap_cgroupseclabel;
static DEFINE_RWLOCK(policy_rwlock);
POLICYDB_CAPABILITY_EXTSOCKCLASS);
selinux_policycap_alwaysnetwork = ebitmap_get_bit(&policydb.policycaps,
POLICYDB_CAPABILITY_ALWAYSNETWORK);
+ selinux_policycap_cgroupseclabel =
+ ebitmap_get_bit(&policydb.policycaps,
+ POLICYDB_CAPABILITY_CGROUPSECLABEL);
}
static int security_preserve_bools(struct policydb *p);
*/
#include "common.h"
+
#include <linux/binfmts.h>
#include <linux/slab.h>
+#include <linux/rculist.h>
/* Variables definitions.*/
*/
#include <linux/slab.h>
+#include <linux/rculist.h>
+
#include "common.h"
/**
*/
#include <linux/slab.h>
+#include <linux/rculist.h>
+
#include "common.h"
/* Lock for protecting policy. */
#include <linux/slab.h>
#include <linux/vmalloc.h>
#include <linux/time.h>
+#include <linux/sched/signal.h>
#include <sound/core.h>
#include <sound/minors.h>
#include <sound/info.h>
#include <linux/time.h>
#include <linux/mutex.h>
#include <linux/module.h>
+#include <linux/sched/signal.h>
#include <sound/core.h>
#include <sound/control.h>
#include <sound/minors.h>
#include <linux/init.h>
#include <linux/slab.h>
+#include <linux/sched/signal.h>
#include <linux/time.h>
#include <linux/vmalloc.h>
#include <linux/module.h>
*/
#include <linux/slab.h>
+#include <linux/sched/signal.h>
#include <linux/time.h>
#include <linux/math64.h>
#include <linux/export.h>
#include <linux/module.h>
#include <linux/file.h>
#include <linux/slab.h>
+#include <linux/sched/signal.h>
#include <linux/time.h>
#include <linux/pm_qos.h>
#include <linux/io.h>
#include <sound/core.h>
#include <linux/major.h>
#include <linux/init.h>
-#include <linux/sched.h>
+#include <linux/sched/signal.h>
#include <linux/slab.h>
#include <linux/time.h>
#include <linux/wait.h>
#include <linux/time.h>
#include <linux/wait.h>
#include <linux/slab.h>
-#include <linux/sched.h>
+#include <linux/sched/signal.h>
#include <sound/core.h>
#include <sound/seq_oss.h>
#include <sound/rawmidi.h>
#include "../seq_clientmgr.h"
#include <linux/wait.h>
#include <linux/slab.h>
+#include <linux/sched/signal.h>
/*
#include <sound/core.h>
#include <linux/slab.h>
+#include <linux/sched/signal.h>
+
#include "seq_fifo.h"
#include "seq_lock.h"
#include <linux/init.h>
#include <linux/export.h>
#include <linux/slab.h>
+#include <linux/sched/signal.h>
#include <linux/vmalloc.h>
#include <sound/core.h>
#include <linux/device.h>
#include <linux/module.h>
#include <linux/string.h>
+#include <linux/sched/signal.h>
#include <sound/core.h>
#include <sound/timer.h>
#include <sound/control.h>
#include <linux/mod_devicetable.h>
#include <linux/delay.h>
#include <linux/slab.h>
+#include <linux/sched/signal.h>
#include <sound/core.h>
#include <sound/initval.h>
#include <linux/slab.h>
#include <linux/spinlock.h>
#include <linux/wait.h>
+#include <linux/sched/signal.h>
#include <sound/control.h>
#include <sound/core.h>
#include <linux/mod_devicetable.h>
#include <linux/delay.h>
#include <linux/slab.h>
+#include <linux/sched/signal.h>
#include <sound/core.h>
#include <sound/initval.h>
#include <linux/mod_devicetable.h>
#include <linux/delay.h>
#include <linux/slab.h>
+#include <linux/sched/signal.h>
#include <sound/core.h>
#include <sound/initval.h>
#include <linux/mutex.h>
#include <linux/slab.h>
#include <linux/compat.h>
+#include <linux/sched/signal.h>
#include <sound/control.h>
#include <sound/core.h>
#include <linux/mutex.h>
#include <linux/slab.h>
#include <linux/compat.h>
+#include <linux/sched/signal.h>
#include <sound/core.h>
#include <sound/initval.h>
#include <asm/dma.h>
#include <linux/slab.h>
+#include <linux/sched/signal.h>
+
#include <sound/core.h>
#include <sound/control.h>
#include <sound/gus.h>
********************************************************************/
#include <linux/kernel.h>
+#include <linux/sched/signal.h>
#include <linux/types.h>
#include <linux/interrupt.h>
#include <linux/io.h>
*/
#include <linux/wait.h>
-#include <linux/sched.h>
+#include <linux/sched/signal.h>
#include <linux/slab.h>
#include <linux/ioport.h>
#include <linux/export.h>
*/
#include "emu8000_local.h"
+
+#include <linux/sched/signal.h>
#include <linux/uaccess.h>
#include <linux/moduleparam.h>
*/
#include "emu8000_local.h"
+
+#include <linux/sched/signal.h>
#include <linux/init.h>
#include <linux/slab.h>
#include <sound/initval.h>
#include <linux/delay.h>
#include <linux/time.h>
#include <linux/wait.h>
+#include <linux/sched/signal.h>
#include <linux/firmware.h>
#include <linux/moduleparam.h>
#include <linux/slab.h>
#include <linux/mm.h>
#include <linux/gfp.h>
+#include <linux/sched/signal.h>
+
#include "sound_config.h"
#include "sleep.h"
#include <linux/soundcard.h>
#include <linux/poll.h>
#include <linux/mutex.h>
+#include <linux/sched/signal.h>
#include <linux/uaccess.h>
#include <linux/stddef.h>
#include <linux/kmod.h>
#include <linux/spinlock.h>
+#include <linux/sched/signal.h>
+
#define MIDIBUF_C
#include "sound_config.h"
#include <linux/interrupt.h>
#include <linux/mutex.h>
#include <linux/gfp.h>
+#include <linux/sched/signal.h>
+
#include <asm/irq.h>
#include <asm/io.h>
#include "sound_config.h"
#include <linux/fs.h>
#include <linux/sound.h>
+#include <linux/sched/signal.h>
#include "os.h"
#include "soundvers.h"
#include <linux/module.h>
#include <linux/string.h>
#include <linux/ioport.h>
-#include <linux/sched.h>
+#include <linux/sched/signal.h>
#include <linux/delay.h>
#include <linux/sound.h>
#include <linux/slab.h>
writel(val, acp_mmio + (reg * 4));
}
-/* Configure a given dma channel parameters - enable/disble,
+/* Configure a given dma channel parameters - enable/disable,
* number of descriptors, priority
*/
static void config_acp_dma_channel(void __iomem *acp_mmio, u8 ch_num,
#ifndef _TOOLS_LINUX_LOG2_H
#define _TOOLS_LINUX_LOG2_H
-/*
- * deal with unrepresentable constant logarithms
- */
-extern __attribute__((const, noreturn))
-int ____ilog2_NaN(void);
-
/*
* non-constant log of base 2 calculators
* - the arch may override these in asm/bitops.h if they can be implemented
#define ilog2(n) \
( \
__builtin_constant_p(n) ? ( \
- (n) < 1 ? ____ilog2_NaN() : \
+ (n) < 2 ? 0 : \
(n) & (1ULL << 63) ? 63 : \
(n) & (1ULL << 62) ? 62 : \
(n) & (1ULL << 61) ? 61 : \
(n) & (1ULL << 4) ? 4 : \
(n) & (1ULL << 3) ? 3 : \
(n) & (1ULL << 2) ? 2 : \
- (n) & (1ULL << 1) ? 1 : \
- (n) & (1ULL << 0) ? 0 : \
- ____ilog2_NaN() \
- ) : \
+ 1 ) : \
(sizeof(n) <= 4) ? \
__ilog2_u32(n) : \
__ilog2_u64(n) \
/* Allow writing to any other BAR, or expansion ROM */
iowrite(portoff, val, mask, &d->config_words[reg]);
return true;
- /* We let them overide latency timer and cacheline size */
+ /* We let them override latency timer and cacheline size */
} else if (&d->config_words[reg] == (void *)&d->config.cacheline_size) {
/* Only let them change the first two fields. */
if (mask == 0xFFFFFFFF)
Q = @
endif
-# Disable command line variables (CFLAGS) overide from top
+# Disable command line variables (CFLAGS) override from top
# level Makefile (perf), otherwise build Makefile will get
# the same command line setup.
MAKEOVERRIDES=
Q = @
endif
-# Disable command line variables (CFLAGS) overide from top
+# Disable command line variables (CFLAGS) override from top
# level Makefile (perf), otherwise build Makefile will get
# the same command line setup.
MAKEOVERRIDES=
* struct pevent_plugin_option PEVENT_PLUGIN_OPTIONS[] = {
* {
* .name = "option-name",
- * .plugin_alias = "overide-file-name", (optional)
+ * .plugin_alias = "override-file-name", (optional)
* .description = "description of option to show users",
* },
* {
insn->jump_dest->offset > orig_insn->offset))
break;
+ /* look for a relocation which references .rodata */
text_rela = find_rela_by_dest_range(insn->sec, insn->offset,
insn->len);
- if (text_rela && text_rela->sym == file->rodata->sym)
- return find_rela_by_dest(file->rodata,
- text_rela->addend);
+ if (!text_rela || text_rela->sym != file->rodata->sym)
+ continue;
+
+ /*
+ * Make sure the .rodata address isn't associated with a
+ * symbol. gcc jump tables are anonymous data.
+ */
+ if (find_symbol_containing(file->rodata, text_rela->addend))
+ continue;
+
+ return find_rela_by_dest(file->rodata, text_rela->addend);
}
return NULL;
return NULL;
}
+struct symbol *find_symbol_containing(struct section *sec, unsigned long offset)
+{
+ struct symbol *sym;
+
+ list_for_each_entry(sym, &sec->symbol_list, list)
+ if (sym->type != STT_SECTION &&
+ offset >= sym->offset && offset < sym->offset + sym->len)
+ return sym;
+
+ return NULL;
+}
+
struct rela *find_rela_by_dest_range(struct section *sec, unsigned long offset,
unsigned int len)
{
struct elf *elf_open(const char *name);
struct section *find_section_by_name(struct elf *elf, const char *name);
struct symbol *find_symbol_by_offset(struct section *sec, unsigned long offset);
+struct symbol *find_symbol_containing(struct section *sec, unsigned long offset);
struct rela *find_rela_by_dest(struct section *sec, unsigned long offset);
struct rela *find_rela_by_dest_range(struct section *sec, unsigned long offset,
unsigned int len);
#error Instruction buffer size too small
#endif
-/* Based on branch_type() from perf_event_intel_lbr.c */
+/* Based on branch_type() from arch/x86/events/intel/lbr.c */
static void intel_pt_insn_decoder(struct insn *insn,
struct intel_pt_insn *intel_pt_insn)
{
There are two ways to invoke turbostat.
The first method is to supply a
\fBcommand\fP, which is forked and statistics are printed
-upon its completion.
+in one-shot upon its completion.
The second method is to omit the command,
-and turbostat displays statistics every 5 seconds.
+and turbostat displays statistics every 5 seconds interval.
The 5-second interval can be changed using the --interval option.
.PP
Some information is not available on older processors.
.PP
\fB--add attributes\fP add column with counter having specified 'attributes'. The 'location' attribute is required, all others are optional.
.nf
- location: {\fBmsrDDD\fP | \fBmsr0xXXX\fP}
+ location: {\fBmsrDDD\fP | \fBmsr0xXXX\fP | \fB/sys/path...\fP}
msrDDD is a decimal offset, eg. msr16
msr0xXXX is a hex offset, eg. msr0x10
+ /sys/path... is an absolute path to a sysfs attribute
scope: {\fBcpu\fP | \fBcore\fP | \fBpackage\fP}
sample and print the counter for every cpu, core, or package.
'delta' shows the difference in values during the measurement interval.
'percent' shows the delta as a percentage of the cycles elapsed.
default: delta
+
+ name: "name_string"
+ Any string that does not match a key-word above is used
+ as the column header.
.fi
.PP
+\fB--cpu cpu-set\fP limit output to system summary plus the specified cpu-set. If cpu-set is the string "core", then the system summary plus the first CPU in each core are printed -- eg. subsequent HT siblings are not printed. Or if cpu-set is the string "package", then the system summary plus the first CPU in each package is printed. Otherwise, the system summary plus the specified set of CPUs are printed. The cpu-set is ordered from low to high, comma delimited with ".." and "-" permitted to denote a range. eg. 1,2,8,14..17,21-44
+.PP
+\fB--hide column\fP do not show the specified columns. May be invoked multiple times, or with a comma-separated list of column names. Use "--hide sysfs" to hide the sysfs statistics columns as a group.
+.PP
+\fB--show column\fP show only the specified columns. May be invoked multiple times, or with a comma-separated list of column names. Use "--show sysfs" to show the sysfs statistics columns as a group.
+.PP
\fB--Dump\fP displays the raw counter values.
.PP
-\fB--debug\fP displays additional system configuration information. Invoking this parameter
-more than once may also enable internal turbostat debug information.
+\fB--quiet\fP Do not decode and print the system configuration header information.
.PP
\fB--interval seconds\fP overrides the default 5.0 second measurement interval.
.PP
.PP
\fB--Joules\fP displays energy in Joules, rather than dividing Joules by time to print power in Watts.
.PP
-\fB--Package\fP limits output to the system summary plus the 1st thread in each Package.
-.PP
-\fB--processor\fP limits output to the system summary plus the 1st thread in each processor of each package. Ie. it skips hyper-threaded siblings.
+\fB--list\fP display column header names available for use by --show and --hide, then exit.
.PP
\fB--Summary\fP limits output to a 1-line System Summary for each interval.
.PP
The \fBcommand\fP parameter forks \fBcommand\fP, and upon its exit,
displays the statistics gathered since it was forked.
.PP
-.SH DEFAULT FIELD DESCRIPTIONS
+.SH ROW DESCRIPTIONS
+The system configuration dump (if --quiet is not used) is followed by statistics. The first row of the statistics labels the content of each column (below). The second row of statistics is the system summary line. The system summary line has a '-' in the columns for the Package, Core, and CPU. The contents of the system summary line depends on the type of column. Columns that count items (eg. IRQ) show the sum across all CPUs in the system. Columns that show a percentage show the average across all CPUs in the system. Columns that dump raw MSR values simply show 0 in the summary. After the system summary row, each row describes a specific Package/Core/CPU. Note that if the --cpu parameter is used to limit which specific CPUs are displayed, turbostat will still collect statistics for all CPUs in the system and will still show the system summary for all CPUs in the system.
+.SH COLUMN DESCRIPTIONS
.nf
+\fBCore\fP processor core number. Note that multiple CPUs per core indicate support for Intel(R) Hyper-Threading Technology (HT).
\fBCPU\fP Linux CPU (logical processor) number. Yes, it is okay that on many systems the CPUs are not listed in numerical order -- for efficiency reasons, turbostat runs in topology order, so HT siblings appear together.
-\fBAVG_MHz\fP number of cycles executed divided by time elapsed.
-\fBBusy%\fP percent of the interval that the CPU retired instructions, aka. % of time in "C0" state.
-\fBBzy_MHz\fP average clock rate while the CPU was busy (in "c0" state).
+\fBPackage\fP processor package number -- not present on systems with a single processor package.
+\fBAvg_MHz\fP number of cycles executed divided by time elapsed. Note that this includes idle-time when 0 instructions are executed.
+\fBBusy%\fP percent of the measurement interval that the CPU executes instructions, aka. % of time in "C0" state.
+\fBBzy_MHz\fP average clock rate while the CPU was not idle (ie. in "c0" state).
\fBTSC_MHz\fP average MHz that the TSC ran during the entire interval.
-.fi
-.PP
-.SH DEBUG FIELD DESCRIPTIONS
-.nf
-\fBPackage\fP processor package number.
-\fBCore\fP processor core number.
-Note that multiple CPUs per core indicate support for Intel(R) Hyper-Threading Technology (HT).
-\fBCPU%c1, CPU%c3, CPU%c6, CPU%c7\fP show the percentage residency in hardware core idle states.
+\fBIRQ\fP The number of interrupts serviced by that CPU during the measurement interval. The system total line is the sum of interrupts serviced across all CPUs. turbostat parses /proc/interrupts to generate this summary.
+\fBSMI\fP The number of System Management Interrupts serviced CPU during the measurement interval. While this counter is actually per-CPU, SMI are triggered on all processors, so the number should be the same for all CPUs.
+\fBC1, C2, C3...\fP The number times Linux requested the C1, C2, C3 idle state during the measurement interval. The system summary line shows the sum for all CPUs. These are C-state names as exported in /sys/devices/system/cpu/cpu*/cpuidle/state*/name. While their names are generic, their attributes are processor specific. They the system description section of output shows what MWAIT sub-states they are mapped to on each system.
+\fBC1%, C2%, C3%\fP The residency percentage that Linux requested C1, C2, C3.... The system summary is the average of all CPUs in the system. Note that these are software, reflecting what was requested. The hardware counters reflect what was actually achieved.
+\fBCPU%c1, CPU%c3, CPU%c6, CPU%c7\fP show the percentage residency in hardware core idle states. These numbers are from hardware residency counters.
\fBCoreTmp\fP Degrees Celsius reported by the per-core Digital Thermal Sensor.
\fBPkgTtmp\fP Degrees Celsius reported by the per-package Package Thermal Monitor.
-\fBPkg%pc2, Pkg%pc3, Pkg%pc6, Pkg%pc7\fP percentage residency in hardware package idle states.
+\fBPkg%pc2, Pkg%pc3, Pkg%pc6, Pkg%pc7\fP percentage residency in hardware package idle states. These numbers are from hardware residency counters.
\fBPkgWatt\fP Watts consumed by the whole package.
\fBCorWatt\fP Watts consumed by the core part of the package.
\fBGFXWatt\fP Watts consumed by the Graphics part of the package -- available only on client processors.
\fBPKG_%\fP percent of the interval that RAPL throttling was active on the Package.
\fBRAM_%\fP percent of the interval that RAPL throttling was active on DRAM.
.fi
+.SH TOO MUCH INFORMATION EXAMPLE
+By default, turbostat dumps all possible information -- a system configuration header, followed by columns for all counters.
+This is ideal for remote debugging, use the "--out" option to save everything to a text file, and get that file to the expert helping you debug.
.PP
-.SH PERIODIC EXAMPLE
-Without any parameters, turbostat displays statistics ever 5 seconds.
-Periodic output goes to stdout, by default, unless --out is used to specify an output file.
-The 5-second interval can be changed with th "-i sec" option.
-Or a command may be specified as in "FORK EXAMPLE" below.
+When you are not interested in all that information, and there are several ways to see only what you want. First the "--quiet" option will skip the configuration information, and turbostat will show only the counter columns. Second, you can reduce the columns with the "--hide" and "--show" options. If you use the "--show" option, then turbostat will show only the columns you list. If you use the "--hide" option, turbostat will show all columns, except the ones you list.
+.PP
+To find out what columns are available for --show and --hide, the "--list" option is available. For convenience, the special strings "sysfs" can be used to refer to all of the sysfs C-state counters at once:
+.nf
+sudo ./turbostat --show sysfs --quiet sleep 10
+10.003837 sec
+ C1 C1E C3 C6 C7s C1% C1E% C3% C6% C7s%
+ 4 21 2 2 459 0.14 0.82 0.00 0.00 98.93
+ 1 17 2 2 130 0.00 0.02 0.00 0.00 99.80
+ 0 0 0 0 31 0.00 0.00 0.00 0.00 99.95
+ 2 1 0 0 52 1.14 6.49 0.00 0.00 92.21
+ 1 2 0 0 52 0.00 0.08 0.00 0.00 99.86
+ 0 0 0 0 71 0.00 0.00 0.00 0.00 99.89
+ 0 0 0 0 25 0.00 0.00 0.00 0.00 99.96
+ 0 0 0 0 74 0.00 0.00 0.00 0.00 99.94
+ 0 1 0 0 24 0.00 0.00 0.00 0.00 99.84
+.fi
+.PP
+.SH ONE SHOT COMMAND EXAMPLE
+If turbostat is invoked with a command, it will fork that command
+and output the statistics gathered after the command exits.
+In this case, turbostat output goes to stderr, by default.
+Output can instead be saved to a file using the --out option.
+In this example, the "sleep 10" command is forked, and turbostat waits for it to complete before saving all statistics into "ts.out". Note that "sleep 10" is not part of turbostat, but is simply an example of a command that turbostat can fork. The "ts.out" file is what you want to edit in a very wide window, paste into a spreadsheet, or attach to a bugzilla entry.
+
.nf
-[root@hsw]# ./turbostat
- CPU Avg_MHz Busy% Bzy_MHz TSC_MHz
- - 488 12.51 3898 3498
- 0 0 0.01 3885 3498
- 4 3897 99.99 3898 3498
- 1 0 0.00 3861 3498
- 5 0 0.00 3882 3498
- 2 1 0.02 3894 3498
- 6 2 0.06 3898 3498
- 3 0 0.00 3849 3498
- 7 0 0.00 3877 3498
+[root@hsw]# ./turbostat -o ts.out sleep 10
+[root@hsw]#
+.fi
+.SH PERIODIC INTERVAL EXAMPLE
+Without a command to fork, turbostat displays statistics ever 5 seconds.
+Periodic output goes to stdout, by default, unless --out is used to specify an output file.
+The 5-second interval can be changed with the "-i sec" option.
+.nf
+sudo ./turbostat --quiet --hide sysfs,IRQ,SMI,CoreTmp,PkgTmp,GFX%rc6,GFXMHz,PkgWatt,CorWatt,GFXWatt
+ Core CPU Avg_MHz Busy% Bzy_MHz TSC_MHz CPU%c1 CPU%c3 CPU%c6 CPU%c7
+ - - 488 12.52 3900 3498 12.50 0.00 0.00 74.98
+ 0 0 5 0.13 3900 3498 99.87 0.00 0.00 0.00
+ 0 4 3897 99.99 3900 3498 0.01
+ 1 1 0 0.00 3856 3498 0.01 0.00 0.00 99.98
+ 1 5 0 0.00 3861 3498 0.01
+ 2 2 1 0.02 3889 3498 0.03 0.00 0.00 99.95
+ 2 6 0 0.00 3863 3498 0.05
+ 3 3 0 0.01 3869 3498 0.02 0.00 0.00 99.97
+ 3 7 0 0.00 3878 3498 0.03
+ Core CPU Avg_MHz Busy% Bzy_MHz TSC_MHz CPU%c1 CPU%c3 CPU%c6 CPU%c7
+ - - 491 12.59 3900 3498 12.42 0.00 0.00 74.99
+ 0 0 27 0.69 3900 3498 99.31 0.00 0.00 0.00
+ 0 4 3898 99.99 3900 3498 0.01
+ 1 1 0 0.00 3883 3498 0.01 0.00 0.00 99.99
+ 1 5 0 0.00 3898 3498 0.01
+ 2 2 0 0.01 3889 3498 0.02 0.00 0.00 99.98
+ 2 6 0 0.00 3889 3498 0.02
+ 3 3 0 0.00 3856 3498 0.01 0.00 0.00 99.99
+ 3 7 0 0.00 3897 3498 0.01
.fi
-.SH DEBUG EXAMPLE
-The "--debug" option prints additional system information before measurements:
+This example also shows the use of the --hide option to skip columns that are not wanted.
+Note that cpu4 in this example is 99.99% busy, while the other CPUs are all under 1% busy.
+Notice that cpu4's HT sibling is cpu0, which is under 1% busy, but can get into CPU%c1 only,
+because its cpu4's activity on shared hardware keeps it from entering a deeper C-state.
-The first row of statistics is a summary for the entire system.
-For residency % columns, the summary is a weighted average.
-For Temperature columns, the summary is the column maximum.
-For Watts columns, the summary is a system total.
-Subsequent rows show per-CPU statistics.
+.SH SYSTEM CONFIGURATION INFORMATION EXAMPLE
+
+By default, turbostat always dumps system configuration information
+before taking measurements. In the example above, "--quiet" is used
+to suppress that output. Here is an example of the configuration information:
.nf
-turbostat version 4.1 10-Feb, 2015 - Len Brown <lenb@kernel.org>
+turbostat version 2017.02.15 - Len Brown <lenb@kernel.org>
CPUID(0): GenuineIntel 13 CPUID levels; family:model:stepping 0x6:3c:3 (6:60:3)
-CPUID(6): APERF, DTS, PTM, EPB
+CPUID(1): SSE3 MONITOR - EIST TM2 TSC MSR ACPI-TM TM
+CPUID(6): APERF, TURBO, DTS, PTM, No-HWP, No-HWPnotify, No-HWPwindow, No-HWPepp, No-HWPpkg, EPB
+cpu4: MSR_IA32_MISC_ENABLE: 0x00850089 (TCC EIST No-MWAIT PREFETCH TURBO)
+CPUID(7): No-SGX
+cpu4: MSR_MISC_PWR_MGMT: 0x00400000 (ENable-EIST_Coordination DISable-EPB DISable-OOB)
RAPL: 3121 sec. Joule Counter Range, at 84 Watts
-cpu0: MSR_NHM_PLATFORM_INFO: 0x80838f3012300
-8 * 100 = 800 MHz max efficiency
-35 * 100 = 3500 MHz TSC frequency
-cpu0: MSR_IA32_POWER_CTL: 0x0004005d (C1E auto-promotion: DISabled)
-cpu0: MSR_NHM_SNB_PKG_CST_CFG_CTL: 0x1e000400 (UNdemote-C3, UNdemote-C1, demote-C3, demote-C1, UNlocked: pkg-cstate-limit=0: pc0)
-cpu0: MSR_TURBO_RATIO_LIMIT: 0x25262727
-37 * 100 = 3700 MHz max turbo 4 active cores
-38 * 100 = 3800 MHz max turbo 3 active cores
-39 * 100 = 3900 MHz max turbo 2 active cores
-39 * 100 = 3900 MHz max turbo 1 active cores
+cpu4: MSR_PLATFORM_INFO: 0x80838f3012300
+8 * 100.0 = 800.0 MHz max efficiency frequency
+35 * 100.0 = 3500.0 MHz base frequency
+cpu4: MSR_IA32_POWER_CTL: 0x0004005d (C1E auto-promotion: DISabled)
+cpu4: MSR_TURBO_RATIO_LIMIT: 0x25262727
+37 * 100.0 = 3700.0 MHz max turbo 4 active cores
+38 * 100.0 = 3800.0 MHz max turbo 3 active cores
+39 * 100.0 = 3900.0 MHz max turbo 2 active cores
+39 * 100.0 = 3900.0 MHz max turbo 1 active cores
+cpu4: MSR_CONFIG_TDP_NOMINAL: 0x00000023 (base_ratio=35)
+cpu4: MSR_CONFIG_TDP_LEVEL_1: 0x00000000 ()
+cpu4: MSR_CONFIG_TDP_LEVEL_2: 0x00000000 ()
+cpu4: MSR_CONFIG_TDP_CONTROL: 0x80000000 ( lock=1)
+cpu4: MSR_TURBO_ACTIVATION_RATIO: 0x00000000 (MAX_NON_TURBO_RATIO=0 lock=0)
+cpu4: MSR_PKG_CST_CONFIG_CONTROL: 0x1e000400 (UNdemote-C3, UNdemote-C1, demote-C3, demote-C1, UNlocked: pkg-cstate-limit=0: pc0)
+cpu4: POLL: CPUIDLE CORE POLL IDLE
+cpu4: C1: MWAIT 0x00
+cpu4: C1E: MWAIT 0x01
+cpu4: C3: MWAIT 0x10
+cpu4: C6: MWAIT 0x20
+cpu4: C7s: MWAIT 0x32
+cpu4: MSR_MISC_FEATURE_CONTROL: 0x00000000 (L2-Prefetch L2-Prefetch-pair L1-Prefetch L1-IP-Prefetch)
cpu0: MSR_IA32_ENERGY_PERF_BIAS: 0x00000006 (balanced)
-cpu0: MSR_CORE_PERF_LIMIT_REASONS, 0x31200000 (Active: ) (Logged: Auto-HWP, Amps, MultiCoreTurbo, Transitions, )
+cpu0: MSR_CORE_PERF_LIMIT_REASONS, 0x31200000 (Active: ) (Logged: Transitions, MultiCoreTurbo, Amps, Auto-HWP, )
cpu0: MSR_GFX_PERF_LIMIT_REASONS, 0x00000000 (Active: ) (Logged: )
cpu0: MSR_RING_PERF_LIMIT_REASONS, 0x0d000000 (Active: ) (Logged: Amps, PkgPwrL1, PkgPwrL2, )
cpu0: MSR_RAPL_POWER_UNIT: 0x000a0e03 (0.125000 Watts, 0.000061 Joules, 0.000977 sec.)
cpu0: MSR_PP1_POWER_LIMIT: 0x00000000 (UNlocked)
cpu0: GFX Limit: DISabled (0.000000 Watts, 0.000977 sec, clamp DISabled)
cpu0: MSR_IA32_TEMPERATURE_TARGET: 0x00641400 (100 C)
-cpu0: MSR_IA32_PACKAGE_THERM_STATUS: 0x88340800 (48 C)
-cpu0: MSR_IA32_THERM_STATUS: 0x88340000 (48 C +/- 1)
-cpu1: MSR_IA32_THERM_STATUS: 0x88440000 (32 C +/- 1)
-cpu2: MSR_IA32_THERM_STATUS: 0x88450000 (31 C +/- 1)
-cpu3: MSR_IA32_THERM_STATUS: 0x88490000 (27 C +/- 1)
- Core CPU Avg_MHz Busy% Bzy_MHz TSC_MHz SMI CPU%c1 CPU%c3 CPU%c6 CPU%c7 CoreTmp PkgTmp PkgWatt CorWatt GFXWatt
- - - 493 12.64 3898 3498 0 12.64 0.00 0.00 74.72 47 47 21.62 13.74 0.00
- 0 0 4 0.11 3894 3498 0 99.89 0.00 0.00 0.00 47 47 21.62 13.74 0.00
- 0 4 3897 99.98 3898 3498 0 0.02
- 1 1 7 0.17 3887 3498 0 0.04 0.00 0.00 99.79 32
- 1 5 0 0.00 3885 3498 0 0.21
- 2 2 29 0.76 3895 3498 0 0.10 0.01 0.01 99.13 32
- 2 6 2 0.06 3896 3498 0 0.80
- 3 3 1 0.02 3832 3498 0 0.03 0.00 0.00 99.95 28
- 3 7 0 0.00 3879 3498 0 0.04
-^C
-
+cpu0: MSR_IA32_PACKAGE_THERM_STATUS: 0x884c0800 (24 C)
+cpu0: MSR_IA32_THERM_STATUS: 0x884c0000 (24 C +/- 1)
+cpu1: MSR_IA32_THERM_STATUS: 0x88510000 (19 C +/- 1)
+cpu2: MSR_IA32_THERM_STATUS: 0x884e0000 (22 C +/- 1)
+cpu3: MSR_IA32_THERM_STATUS: 0x88510000 (19 C +/- 1)
+cpu4: MSR_PKGC3_IRTL: 0x00008842 (valid, 67584 ns)
+cpu4: MSR_PKGC6_IRTL: 0x00008873 (valid, 117760 ns)
+cpu4: MSR_PKGC7_IRTL: 0x00008891 (valid, 148480 ns)
.fi
The \fBmax efficiency\fP frequency, a.k.a. Low Frequency Mode, is the frequency
available at the minimum package voltage. The \fBTSC frequency\fP is the base
The remaining rows show what maximum turbo frequency is possible
depending on the number of idle cores. Note that not all information is
available on all processors.
-.PP
-The --debug option adds additional columns to the measurement ouput, including CPU idle power-state residency processor temperature sensor readinds.
-See the field definitions above.
-.SH FORK EXAMPLE
-If turbostat is invoked with a command, it will fork that command
-and output the statistics gathered after the command exits.
-In this case, turbostat output goes to stderr, by default.
-Output can instead be saved to a file using the --out option.
-eg. Here a cycle soaker is run on 1 CPU (see %c0) for a few seconds
-until ^C while the other CPUs are mostly idle:
-
+.SH ADD COUNTER EXAMPLE
+Here we limit turbostat to showing just the CPU number for cpu0 - cpu3.
+We add a counter showing the 32-bit raw value of MSR 0x199 (MSR_IA32_PERF_CTL),
+labeling it with the column header, "PRF_CTRL", and display it only once,
+afte the conclusion of a 0.1 second sleep.
.nf
-root@hsw: turbostat cat /dev/zero > /dev/null
-^C
- CPU Avg_MHz Busy% Bzy_MHz TSC_MHz
- - 482 12.51 3854 3498
- 0 0 0.01 1960 3498
- 4 0 0.00 2128 3498
- 1 0 0.00 3003 3498
- 5 3854 99.98 3855 3498
- 2 0 0.01 3504 3498
- 6 3 0.08 3884 3498
- 3 0 0.00 2553 3498
- 7 0 0.00 2126 3498
-10.783983 sec
+sudo ./turbostat --quiet --cpu 0-3 --show CPU --add msr0x199,u32,raw,PRF_CTRL sleep .1
+0.101604 sec
+CPU PRF_CTRL
+- 0x00000000
+0 0x00000c00
+1 0x00000800
+2 0x00000a00
+3 0x00000800
.fi
-Above the cycle soaker drives cpu5 up its 3.9 GHz turbo limit.
-The first row shows the average MHz and Busy% across all the processors in the system.
-
-Note that the Avg_MHz column reflects the total number of cycles executed
-divided by the measurement interval. If the Busy% column is 100%,
-then the processor was running at that speed the entire interval.
-The Avg_MHz multiplied by the Busy% results in the Bzy_MHz --
-which is the average frequency while the processor was executing --
-not including any non-busy idle time.
.SH NOTES
int *fd_percpu;
struct timespec interval_ts = {5, 0};
unsigned int debug;
+unsigned int quiet;
+unsigned int sums_need_wide_columns;
unsigned int rapl_joules;
unsigned int summary_only;
+unsigned int list_header_only;
unsigned int dump_only;
-unsigned int do_nhm_cstates;
unsigned int do_snb_cstates;
unsigned int do_knl_cstates;
-unsigned int do_pc2;
-unsigned int do_pc3;
-unsigned int do_pc6;
-unsigned int do_pc7;
-unsigned int do_c8_c9_c10;
unsigned int do_skl_residency;
unsigned int do_slm_cstates;
unsigned int use_c1_residency_msr;
unsigned int genuine_intel;
unsigned int has_invariant_tsc;
unsigned int do_nhm_platform_info;
+unsigned int no_MSR_MISC_PWR_MGMT;
unsigned int aperf_mperf_multiplier = 1;
-int do_irq = 1;
-int do_smi;
double bclk;
double base_hz;
unsigned int has_base_hz;
double tsc_tweak = 1.0;
-unsigned int show_pkg;
-unsigned int show_core;
-unsigned int show_cpu;
unsigned int show_pkg_only;
unsigned int show_core_only;
char *output_buffer, *outp;
unsigned int do_rapl;
unsigned int do_dts;
unsigned int do_ptm;
-unsigned int do_gfx_rc6_ms;
unsigned long long gfx_cur_rc6_ms;
-unsigned int do_gfx_mhz;
unsigned int gfx_cur_mhz;
unsigned int tcc_activation_temp;
unsigned int tcc_activation_temp_override;
unsigned int has_hwp_activity_window; /* IA32_HWP_REQUEST[bits 41:32] */
unsigned int has_hwp_epp; /* IA32_HWP_REQUEST[bits 31:24] */
unsigned int has_hwp_pkg; /* IA32_HWP_REQUEST_PKG */
+unsigned int has_misc_feature_control;
#define RAPL_PKG (1 << 0)
/* 0x610 MSR_PKG_POWER_LIMIT */
* Usually truncated to 7 characters, but also handles 18 columns for raw 64-bit counters
*/
#define NAME_BYTES 20
+#define PATH_BYTES 128
int backwards_count;
char *progname;
-cpu_set_t *cpu_present_set, *cpu_affinity_set;
-size_t cpu_present_setsize, cpu_affinity_setsize;
+#define CPU_SUBSET_MAXCPUS 1024 /* need to use before probe... */
+cpu_set_t *cpu_present_set, *cpu_affinity_set, *cpu_subset;
+size_t cpu_present_setsize, cpu_affinity_setsize, cpu_subset_size;
+#define MAX_ADDED_COUNTERS 16
struct thread_data {
unsigned long long tsc;
unsigned long long aperf;
unsigned long long mperf;
unsigned long long c1;
- unsigned int irq_count;
+ unsigned long long irq_count;
unsigned int smi_count;
unsigned int cpu_id;
unsigned int flags;
#define CPU_IS_FIRST_THREAD_IN_CORE 0x2
#define CPU_IS_FIRST_CORE_IN_PACKAGE 0x4
- unsigned long long counter[1];
+ unsigned long long counter[MAX_ADDED_COUNTERS];
} *thread_even, *thread_odd;
struct core_data {
unsigned long long c3;
unsigned long long c6;
unsigned long long c7;
+ unsigned long long mc6_us; /* duplicate as per-core for now, even though per module */
unsigned int core_temp_c;
unsigned int core_id;
- unsigned long long counter[1];
+ unsigned long long counter[MAX_ADDED_COUNTERS];
} *core_even, *core_odd;
struct pkg_data {
unsigned int rapl_pkg_perf_status; /* MSR_PKG_PERF_STATUS */
unsigned int rapl_dram_perf_status; /* MSR_DRAM_PERF_STATUS */
unsigned int pkg_temp_c;
- unsigned long long counter[1];
+ unsigned long long counter[MAX_ADDED_COUNTERS];
} *package_even, *package_odd;
#define ODD_COUNTERS thread_odd, core_odd, package_odd
#define GET_PKG(pkg_base, pkg_no) (pkg_base + pkg_no)
enum counter_scope {SCOPE_CPU, SCOPE_CORE, SCOPE_PACKAGE};
-enum counter_type {COUNTER_CYCLES, COUNTER_SECONDS};
+enum counter_type {COUNTER_ITEMS, COUNTER_CYCLES, COUNTER_SECONDS, COUNTER_USEC};
enum counter_format {FORMAT_RAW, FORMAT_DELTA, FORMAT_PERCENT};
struct msr_counter {
unsigned int msr_num;
char name[NAME_BYTES];
+ char path[PATH_BYTES];
unsigned int width;
enum counter_type type;
enum counter_format format;
struct msr_counter *next;
+ unsigned int flags;
+#define FLAGS_HIDE (1 << 0)
+#define FLAGS_SHOW (1 << 1)
+#define SYSFS_PERCPU (1 << 1)
};
struct sys_counters {
- unsigned int thread_counter_bytes;
- unsigned int core_counter_bytes;
- unsigned int package_counter_bytes;
+ unsigned int added_thread_counters;
+ unsigned int added_core_counters;
+ unsigned int added_package_counters;
struct msr_counter *tp;
struct msr_counter *cp;
struct msr_counter *pp;
retval = pread(get_msr_fd(cpu), msr, sizeof(*msr), offset);
if (retval != sizeof *msr)
- err(-1, "msr %d offset 0x%llx read failed", cpu, (unsigned long long)offset);
+ err(-1, "cpu%d: msr offset 0x%llx read failed", cpu, (unsigned long long)offset);
return 0;
}
/*
- * Example Format w/ field column widths:
- *
- * Package Core CPU Avg_MHz Bzy_MHz TSC_MHz IRQ SMI Busy% CPU_%c1 CPU_%c3 CPU_%c6 CPU_%c7 ThreadC CoreTmp CoreCnt PkgTmp GFXMHz Pkg%pc2 Pkg%pc3 Pkg%pc6 Pkg%pc7 PkgWatt CorWatt GFXWatt PkgCnt
- * 12345678123456781234567812345678123456781234567812345678123456781234567812345678123456781234567812345678123456781234567812345678123456781234567812345678123456781234567812345678
+ * Each string in this array is compared in --show and --hide cmdline.
+ * Thus, strings that are proper sub-sets must follow their more specific peers.
+ */
+struct msr_counter bic[] = {
+ { 0x0, "Package" },
+ { 0x0, "Avg_MHz" },
+ { 0x0, "Bzy_MHz" },
+ { 0x0, "TSC_MHz" },
+ { 0x0, "IRQ" },
+ { 0x0, "SMI", "", 32, 0, FORMAT_DELTA, NULL},
+ { 0x0, "Busy%" },
+ { 0x0, "CPU%c1" },
+ { 0x0, "CPU%c3" },
+ { 0x0, "CPU%c6" },
+ { 0x0, "CPU%c7" },
+ { 0x0, "ThreadC" },
+ { 0x0, "CoreTmp" },
+ { 0x0, "CoreCnt" },
+ { 0x0, "PkgTmp" },
+ { 0x0, "GFX%rc6" },
+ { 0x0, "GFXMHz" },
+ { 0x0, "Pkg%pc2" },
+ { 0x0, "Pkg%pc3" },
+ { 0x0, "Pkg%pc6" },
+ { 0x0, "Pkg%pc7" },
+ { 0x0, "Pkg%pc8" },
+ { 0x0, "Pkg%pc9" },
+ { 0x0, "Pkg%pc10" },
+ { 0x0, "PkgWatt" },
+ { 0x0, "CorWatt" },
+ { 0x0, "GFXWatt" },
+ { 0x0, "PkgCnt" },
+ { 0x0, "RAMWatt" },
+ { 0x0, "PKG_%" },
+ { 0x0, "RAM_%" },
+ { 0x0, "Pkg_J" },
+ { 0x0, "Cor_J" },
+ { 0x0, "GFX_J" },
+ { 0x0, "RAM_J" },
+ { 0x0, "Core" },
+ { 0x0, "CPU" },
+ { 0x0, "Mod%c6" },
+ { 0x0, "sysfs" },
+};
+
+#define MAX_BIC (sizeof(bic) / sizeof(struct msr_counter))
+#define BIC_Package (1ULL << 0)
+#define BIC_Avg_MHz (1ULL << 1)
+#define BIC_Bzy_MHz (1ULL << 2)
+#define BIC_TSC_MHz (1ULL << 3)
+#define BIC_IRQ (1ULL << 4)
+#define BIC_SMI (1ULL << 5)
+#define BIC_Busy (1ULL << 6)
+#define BIC_CPU_c1 (1ULL << 7)
+#define BIC_CPU_c3 (1ULL << 8)
+#define BIC_CPU_c6 (1ULL << 9)
+#define BIC_CPU_c7 (1ULL << 10)
+#define BIC_ThreadC (1ULL << 11)
+#define BIC_CoreTmp (1ULL << 12)
+#define BIC_CoreCnt (1ULL << 13)
+#define BIC_PkgTmp (1ULL << 14)
+#define BIC_GFX_rc6 (1ULL << 15)
+#define BIC_GFXMHz (1ULL << 16)
+#define BIC_Pkgpc2 (1ULL << 17)
+#define BIC_Pkgpc3 (1ULL << 18)
+#define BIC_Pkgpc6 (1ULL << 19)
+#define BIC_Pkgpc7 (1ULL << 20)
+#define BIC_Pkgpc8 (1ULL << 21)
+#define BIC_Pkgpc9 (1ULL << 22)
+#define BIC_Pkgpc10 (1ULL << 23)
+#define BIC_PkgWatt (1ULL << 24)
+#define BIC_CorWatt (1ULL << 25)
+#define BIC_GFXWatt (1ULL << 26)
+#define BIC_PkgCnt (1ULL << 27)
+#define BIC_RAMWatt (1ULL << 28)
+#define BIC_PKG__ (1ULL << 29)
+#define BIC_RAM__ (1ULL << 30)
+#define BIC_Pkg_J (1ULL << 31)
+#define BIC_Cor_J (1ULL << 32)
+#define BIC_GFX_J (1ULL << 33)
+#define BIC_RAM_J (1ULL << 34)
+#define BIC_Core (1ULL << 35)
+#define BIC_CPU (1ULL << 36)
+#define BIC_Mod_c6 (1ULL << 37)
+#define BIC_sysfs (1ULL << 38)
+
+unsigned long long bic_enabled = 0xFFFFFFFFFFFFFFFFULL;
+unsigned long long bic_present = BIC_sysfs;
+
+#define DO_BIC(COUNTER_NAME) (bic_enabled & bic_present & COUNTER_NAME)
+#define BIC_PRESENT(COUNTER_BIT) (bic_present |= COUNTER_BIT)
+#define BIC_NOT_PRESENT(COUNTER_BIT) (bic_present &= ~COUNTER_BIT)
+
+#define MAX_DEFERRED 16
+char *deferred_skip_names[MAX_DEFERRED];
+int deferred_skip_index;
+
+/*
+ * HIDE_LIST - hide this list of counters, show the rest [default]
+ * SHOW_LIST - show this list of counters, hide the rest
*/
+enum show_hide_mode { SHOW_LIST, HIDE_LIST } global_show_hide_mode = HIDE_LIST;
-void print_header(void)
+void help(void)
{
- struct msr_counter *mp;
+ fprintf(outf,
+ "Usage: turbostat [OPTIONS][(--interval seconds) | COMMAND ...]\n"
+ "\n"
+ "Turbostat forks the specified COMMAND and prints statistics\n"
+ "when COMMAND completes.\n"
+ "If no COMMAND is specified, turbostat wakes every 5-seconds\n"
+ "to print statistics, until interrupted.\n"
+ "--add add a counter\n"
+ " eg. --add msr0x10,u64,cpu,delta,MY_TSC\n"
+ "--cpu cpu-set limit output to summary plus cpu-set:\n"
+ " {core | package | j,k,l..m,n-p }\n"
+ "--quiet skip decoding system configuration header\n"
+ "--interval sec Override default 5-second measurement interval\n"
+ "--help print this help message\n"
+ "--list list column headers only\n"
+ "--out file create or truncate \"file\" for all output\n"
+ "--version print version information\n"
+ "\n"
+ "For more help, run \"man turbostat\"\n");
+}
- if (show_pkg)
- outp += sprintf(outp, "\tPackage");
- if (show_core)
- outp += sprintf(outp, "\tCore");
- if (show_cpu)
- outp += sprintf(outp, "\tCPU");
- if (has_aperf)
- outp += sprintf(outp, "\tAvg_MHz");
- if (has_aperf)
- outp += sprintf(outp, "\tBusy%%");
- if (has_aperf)
- outp += sprintf(outp, "\tBzy_MHz");
- outp += sprintf(outp, "\tTSC_MHz");
+/*
+ * bic_lookup
+ * for all the strings in comma separate name_list,
+ * set the approprate bit in return value.
+ */
+unsigned long long bic_lookup(char *name_list, enum show_hide_mode mode)
+{
+ int i;
+ unsigned long long retval = 0;
- if (!debug)
- goto done;
+ while (name_list) {
+ char *comma;
- if (do_irq)
- outp += sprintf(outp, "\tIRQ");
- if (do_smi)
- outp += sprintf(outp, "\tSMI");
-
- if (do_nhm_cstates)
- outp += sprintf(outp, "\tCPU%%c1");
- if (do_nhm_cstates && !do_slm_cstates && !do_knl_cstates)
- outp += sprintf(outp, "\tCPU%%c3");
- if (do_nhm_cstates)
- outp += sprintf(outp, "\tCPU%%c6");
- if (do_snb_cstates)
- outp += sprintf(outp, "\tCPU%%c7");
+ comma = strchr(name_list, ',');
+
+ if (comma)
+ *comma = '\0';
+
+ for (i = 0; i < MAX_BIC; ++i) {
+ if (!strcmp(name_list, bic[i].name)) {
+ retval |= (1ULL << i);
+ break;
+ }
+ }
+ if (i == MAX_BIC) {
+ if (mode == SHOW_LIST) {
+ fprintf(stderr, "Invalid counter name: %s\n", name_list);
+ exit(-1);
+ }
+ deferred_skip_names[deferred_skip_index++] = name_list;
+ if (debug)
+ fprintf(stderr, "deferred \"%s\"\n", name_list);
+ if (deferred_skip_index >= MAX_DEFERRED) {
+ fprintf(stderr, "More than max %d un-recognized --skip options '%s'\n",
+ MAX_DEFERRED, name_list);
+ help();
+ exit(1);
+ }
+ }
+
+ name_list = comma;
+ if (name_list)
+ name_list++;
+
+ }
+ return retval;
+}
+
+
+void print_header(char *delim)
+{
+ struct msr_counter *mp;
+ int printed = 0;
+
+ if (DO_BIC(BIC_Package))
+ outp += sprintf(outp, "%sPackage", (printed++ ? delim : ""));
+ if (DO_BIC(BIC_Core))
+ outp += sprintf(outp, "%sCore", (printed++ ? delim : ""));
+ if (DO_BIC(BIC_CPU))
+ outp += sprintf(outp, "%sCPU", (printed++ ? delim : ""));
+ if (DO_BIC(BIC_Avg_MHz))
+ outp += sprintf(outp, "%sAvg_MHz", (printed++ ? delim : ""));
+ if (DO_BIC(BIC_Busy))
+ outp += sprintf(outp, "%sBusy%%", (printed++ ? delim : ""));
+ if (DO_BIC(BIC_Bzy_MHz))
+ outp += sprintf(outp, "%sBzy_MHz", (printed++ ? delim : ""));
+ if (DO_BIC(BIC_TSC_MHz))
+ outp += sprintf(outp, "%sTSC_MHz", (printed++ ? delim : ""));
+
+ if (DO_BIC(BIC_IRQ)) {
+ if (sums_need_wide_columns)
+ outp += sprintf(outp, "%s IRQ", (printed++ ? delim : ""));
+ else
+ outp += sprintf(outp, "%sIRQ", (printed++ ? delim : ""));
+ }
+
+ if (DO_BIC(BIC_SMI))
+ outp += sprintf(outp, "%sSMI", (printed++ ? delim : ""));
for (mp = sys.tp; mp; mp = mp->next) {
+
if (mp->format == FORMAT_RAW) {
if (mp->width == 64)
- outp += sprintf(outp, "\t%18.18s", mp->name);
+ outp += sprintf(outp, "%s%18.18s", (printed++ ? delim : ""), mp->name);
else
- outp += sprintf(outp, "\t%10.10s", mp->name);
+ outp += sprintf(outp, "%s%10.10s", (printed++ ? delim : ""), mp->name);
} else {
- outp += sprintf(outp, "\t%-7.7s", mp->name);
+ if ((mp->type == COUNTER_ITEMS) && sums_need_wide_columns)
+ outp += sprintf(outp, "%s%8s", (printed++ ? delim : ""), mp->name);
+ else
+ outp += sprintf(outp, "%s%s", (printed++ ? delim : ""), mp->name);
}
}
- if (do_dts)
- outp += sprintf(outp, "\tCoreTmp");
+ if (DO_BIC(BIC_CPU_c1))
+ outp += sprintf(outp, "%sCPU%%c1", (printed++ ? delim : ""));
+ if (DO_BIC(BIC_CPU_c3) && !do_slm_cstates && !do_knl_cstates)
+ outp += sprintf(outp, "%sCPU%%c3", (printed++ ? delim : ""));
+ if (DO_BIC(BIC_CPU_c6))
+ outp += sprintf(outp, "%sCPU%%c6", (printed++ ? delim : ""));
+ if (DO_BIC(BIC_CPU_c7))
+ outp += sprintf(outp, "%sCPU%%c7", (printed++ ? delim : ""));
+
+ if (DO_BIC(BIC_Mod_c6))
+ outp += sprintf(outp, "%sMod%%c6", (printed++ ? delim : ""));
+
+ if (DO_BIC(BIC_CoreTmp))
+ outp += sprintf(outp, "%sCoreTmp", (printed++ ? delim : ""));
for (mp = sys.cp; mp; mp = mp->next) {
if (mp->format == FORMAT_RAW) {
if (mp->width == 64)
- outp += sprintf(outp, "\t%18.18s", mp->name);
+ outp += sprintf(outp, "%s%18.18s", delim, mp->name);
else
- outp += sprintf(outp, "\t%10.10s", mp->name);
+ outp += sprintf(outp, "%s%10.10s", delim, mp->name);
} else {
- outp += sprintf(outp, "\t%-7.7s", mp->name);
+ if ((mp->type == COUNTER_ITEMS) && sums_need_wide_columns)
+ outp += sprintf(outp, "%s%8s", delim, mp->name);
+ else
+ outp += sprintf(outp, "%s%s", delim, mp->name);
}
}
- if (do_ptm)
- outp += sprintf(outp, "\tPkgTmp");
+ if (DO_BIC(BIC_PkgTmp))
+ outp += sprintf(outp, "%sPkgTmp", (printed++ ? delim : ""));
- if (do_gfx_rc6_ms)
- outp += sprintf(outp, "\tGFX%%rc6");
+ if (DO_BIC(BIC_GFX_rc6))
+ outp += sprintf(outp, "%sGFX%%rc6", (printed++ ? delim : ""));
- if (do_gfx_mhz)
- outp += sprintf(outp, "\tGFXMHz");
+ if (DO_BIC(BIC_GFXMHz))
+ outp += sprintf(outp, "%sGFXMHz", (printed++ ? delim : ""));
if (do_skl_residency) {
- outp += sprintf(outp, "\tTotl%%C0");
- outp += sprintf(outp, "\tAny%%C0");
- outp += sprintf(outp, "\tGFX%%C0");
- outp += sprintf(outp, "\tCPUGFX%%");
- }
-
- if (do_pc2)
- outp += sprintf(outp, "\tPkg%%pc2");
- if (do_pc3)
- outp += sprintf(outp, "\tPkg%%pc3");
- if (do_pc6)
- outp += sprintf(outp, "\tPkg%%pc6");
- if (do_pc7)
- outp += sprintf(outp, "\tPkg%%pc7");
- if (do_c8_c9_c10) {
- outp += sprintf(outp, "\tPkg%%pc8");
- outp += sprintf(outp, "\tPkg%%pc9");
- outp += sprintf(outp, "\tPk%%pc10");
+ outp += sprintf(outp, "%sTotl%%C0", (printed++ ? delim : ""));
+ outp += sprintf(outp, "%sAny%%C0", (printed++ ? delim : ""));
+ outp += sprintf(outp, "%sGFX%%C0", (printed++ ? delim : ""));
+ outp += sprintf(outp, "%sCPUGFX%%", (printed++ ? delim : ""));
}
+ if (DO_BIC(BIC_Pkgpc2))
+ outp += sprintf(outp, "%sPkg%%pc2", (printed++ ? delim : ""));
+ if (DO_BIC(BIC_Pkgpc3))
+ outp += sprintf(outp, "%sPkg%%pc3", (printed++ ? delim : ""));
+ if (DO_BIC(BIC_Pkgpc6))
+ outp += sprintf(outp, "%sPkg%%pc6", (printed++ ? delim : ""));
+ if (DO_BIC(BIC_Pkgpc7))
+ outp += sprintf(outp, "%sPkg%%pc7", (printed++ ? delim : ""));
+ if (DO_BIC(BIC_Pkgpc8))
+ outp += sprintf(outp, "%sPkg%%pc8", (printed++ ? delim : ""));
+ if (DO_BIC(BIC_Pkgpc9))
+ outp += sprintf(outp, "%sPkg%%pc9", (printed++ ? delim : ""));
+ if (DO_BIC(BIC_Pkgpc10))
+ outp += sprintf(outp, "%sPk%%pc10", (printed++ ? delim : ""));
+
if (do_rapl && !rapl_joules) {
- if (do_rapl & RAPL_PKG)
- outp += sprintf(outp, "\tPkgWatt");
- if (do_rapl & RAPL_CORES_ENERGY_STATUS)
- outp += sprintf(outp, "\tCorWatt");
- if (do_rapl & RAPL_GFX)
- outp += sprintf(outp, "\tGFXWatt");
- if (do_rapl & RAPL_DRAM)
- outp += sprintf(outp, "\tRAMWatt");
- if (do_rapl & RAPL_PKG_PERF_STATUS)
- outp += sprintf(outp, "\tPKG_%%");
- if (do_rapl & RAPL_DRAM_PERF_STATUS)
- outp += sprintf(outp, "\tRAM_%%");
+ if (DO_BIC(BIC_PkgWatt))
+ outp += sprintf(outp, "%sPkgWatt", (printed++ ? delim : ""));
+ if (DO_BIC(BIC_CorWatt))
+ outp += sprintf(outp, "%sCorWatt", (printed++ ? delim : ""));
+ if (DO_BIC(BIC_GFXWatt))
+ outp += sprintf(outp, "%sGFXWatt", (printed++ ? delim : ""));
+ if (DO_BIC(BIC_RAMWatt))
+ outp += sprintf(outp, "%sRAMWatt", (printed++ ? delim : ""));
+ if (DO_BIC(BIC_PKG__))
+ outp += sprintf(outp, "%sPKG_%%", (printed++ ? delim : ""));
+ if (DO_BIC(BIC_RAM__))
+ outp += sprintf(outp, "%sRAM_%%", (printed++ ? delim : ""));
} else if (do_rapl && rapl_joules) {
- if (do_rapl & RAPL_PKG)
- outp += sprintf(outp, "\tPkg_J");
- if (do_rapl & RAPL_CORES_ENERGY_STATUS)
- outp += sprintf(outp, "\tCor_J");
- if (do_rapl & RAPL_GFX)
- outp += sprintf(outp, "\tGFX_J");
- if (do_rapl & RAPL_DRAM)
- outp += sprintf(outp, "\tRAM_J");
- if (do_rapl & RAPL_PKG_PERF_STATUS)
- outp += sprintf(outp, "\tPKG_%%");
- if (do_rapl & RAPL_DRAM_PERF_STATUS)
- outp += sprintf(outp, "\tRAM_%%");
+ if (DO_BIC(BIC_Pkg_J))
+ outp += sprintf(outp, "%sPkg_J", (printed++ ? delim : ""));
+ if (DO_BIC(BIC_Cor_J))
+ outp += sprintf(outp, "%sCor_J", (printed++ ? delim : ""));
+ if (DO_BIC(BIC_GFX_J))
+ outp += sprintf(outp, "%sGFX_J", (printed++ ? delim : ""));
+ if (DO_BIC(BIC_RAM_J))
+ outp += sprintf(outp, "%sRAM_J", (printed++ ? delim : ""));
+ if (DO_BIC(BIC_PKG__))
+ outp += sprintf(outp, "%sPKG_%%", (printed++ ? delim : ""));
+ if (DO_BIC(BIC_RAM__))
+ outp += sprintf(outp, "%sRAM_%%", (printed++ ? delim : ""));
}
for (mp = sys.pp; mp; mp = mp->next) {
if (mp->format == FORMAT_RAW) {
if (mp->width == 64)
- outp += sprintf(outp, "\t%18.18s", mp->name);
+ outp += sprintf(outp, "%s%18.18s", delim, mp->name);
else
- outp += sprintf(outp, "\t%10.10s", mp->name);
+ outp += sprintf(outp, "%s%10.10s", delim, mp->name);
} else {
- outp += sprintf(outp, "\t%-7.7s", mp->name);
+ if ((mp->type == COUNTER_ITEMS) && sums_need_wide_columns)
+ outp += sprintf(outp, "%s%8s", delim, mp->name);
+ else
+ outp += sprintf(outp, "%s%s", delim, mp->name);
}
}
-done:
outp += sprintf(outp, "\n");
}
outp += sprintf(outp, "mperf: %016llX\n", t->mperf);
outp += sprintf(outp, "c1: %016llX\n", t->c1);
- if (do_irq)
- outp += sprintf(outp, "IRQ: %08X\n", t->irq_count);
- if (do_smi)
- outp += sprintf(outp, "SMI: %08X\n", t->smi_count);
+ if (DO_BIC(BIC_IRQ))
+ outp += sprintf(outp, "IRQ: %lld\n", t->irq_count);
+ if (DO_BIC(BIC_SMI))
+ outp += sprintf(outp, "SMI: %d\n", t->smi_count);
for (i = 0, mp = sys.tp; mp; i++, mp = mp->next) {
outp += sprintf(outp, "tADDED [%d] msr0x%x: %08llX\n",
outp += sprintf(outp, "cADDED [%d] msr0x%x: %08llX\n",
i, mp->msr_num, c->counter[i]);
}
+ outp += sprintf(outp, "mc6_us: %016llX\n", c->mc6_us);
}
if (p) {
outp += sprintf(outp, "CPU + GFX: %016llX\n", p->pkg_both_core_gfxe_c0);
outp += sprintf(outp, "pc2: %016llX\n", p->pc2);
- if (do_pc3)
+ if (DO_BIC(BIC_Pkgpc3))
outp += sprintf(outp, "pc3: %016llX\n", p->pc3);
- if (do_pc6)
+ if (DO_BIC(BIC_Pkgpc6))
outp += sprintf(outp, "pc6: %016llX\n", p->pc6);
- if (do_pc7)
+ if (DO_BIC(BIC_Pkgpc7))
outp += sprintf(outp, "pc7: %016llX\n", p->pc7);
outp += sprintf(outp, "pc8: %016llX\n", p->pc8);
outp += sprintf(outp, "pc9: %016llX\n", p->pc9);
int format_counters(struct thread_data *t, struct core_data *c,
struct pkg_data *p)
{
- double interval_float;
+ double interval_float, tsc;
char *fmt8;
int i;
struct msr_counter *mp;
+ char *delim = "\t";
+ int printed = 0;
/* if showing only 1st thread in core and this isn't one, bail out */
if (show_core_only && !(t->flags & CPU_IS_FIRST_THREAD_IN_CORE))
if (show_pkg_only && !(t->flags & CPU_IS_FIRST_CORE_IN_PACKAGE))
return 0;
+ /*if not summary line and --cpu is used */
+ if ((t != &average.threads) &&
+ (cpu_subset && !CPU_ISSET_S(t->cpu_id, cpu_subset_size, cpu_subset)))
+ return 0;
+
interval_float = tv_delta.tv_sec + tv_delta.tv_usec/1000000.0;
+ tsc = t->tsc * tsc_tweak;
+
/* topo columns, print blanks on 1st (average) line */
if (t == &average.threads) {
- if (show_pkg)
- outp += sprintf(outp, "\t-");
- if (show_core)
- outp += sprintf(outp, "\t-");
- if (show_cpu)
- outp += sprintf(outp, "\t-");
+ if (DO_BIC(BIC_Package))
+ outp += sprintf(outp, "%s-", (printed++ ? delim : ""));
+ if (DO_BIC(BIC_Core))
+ outp += sprintf(outp, "%s-", (printed++ ? delim : ""));
+ if (DO_BIC(BIC_CPU))
+ outp += sprintf(outp, "%s-", (printed++ ? delim : ""));
} else {
- if (show_pkg) {
+ if (DO_BIC(BIC_Package)) {
if (p)
- outp += sprintf(outp, "\t%d", p->package_id);
+ outp += sprintf(outp, "%s%d", (printed++ ? delim : ""), p->package_id);
else
- outp += sprintf(outp, "\t-");
+ outp += sprintf(outp, "%s-", (printed++ ? delim : ""));
}
- if (show_core) {
+ if (DO_BIC(BIC_Core)) {
if (c)
- outp += sprintf(outp, "\t%d", c->core_id);
+ outp += sprintf(outp, "%s%d", (printed++ ? delim : ""), c->core_id);
else
- outp += sprintf(outp, "\t-");
+ outp += sprintf(outp, "%s-", (printed++ ? delim : ""));
}
- if (show_cpu)
- outp += sprintf(outp, "\t%d", t->cpu_id);
+ if (DO_BIC(BIC_CPU))
+ outp += sprintf(outp, "%s%d", (printed++ ? delim : ""), t->cpu_id);
}
- /* Avg_MHz */
- if (has_aperf)
- outp += sprintf(outp, "\t%.0f",
+ if (DO_BIC(BIC_Avg_MHz))
+ outp += sprintf(outp, "%s%.0f", (printed++ ? delim : ""),
1.0 / units * t->aperf / interval_float);
- /* Busy% */
- if (has_aperf)
- outp += sprintf(outp, "\t%.2f", 100.0 * t->mperf/t->tsc/tsc_tweak);
+ if (DO_BIC(BIC_Busy))
+ outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), 100.0 * t->mperf/tsc);
- /* Bzy_MHz */
- if (has_aperf) {
+ if (DO_BIC(BIC_Bzy_MHz)) {
if (has_base_hz)
- outp += sprintf(outp, "\t%.0f", base_hz / units * t->aperf / t->mperf);
+ outp += sprintf(outp, "%s%.0f", (printed++ ? delim : ""), base_hz / units * t->aperf / t->mperf);
else
- outp += sprintf(outp, "\t%.0f",
- 1.0 * t->tsc / units * t->aperf / t->mperf / interval_float);
+ outp += sprintf(outp, "%s%.0f", (printed++ ? delim : ""),
+ tsc / units * t->aperf / t->mperf / interval_float);
}
- /* TSC_MHz */
- outp += sprintf(outp, "\t%.0f", 1.0 * t->tsc/units/interval_float);
-
- if (!debug)
- goto done;
+ if (DO_BIC(BIC_TSC_MHz))
+ outp += sprintf(outp, "%s%.0f", (printed++ ? delim : ""), 1.0 * t->tsc/units/interval_float);
/* IRQ */
- if (do_irq)
- outp += sprintf(outp, "\t%d", t->irq_count);
+ if (DO_BIC(BIC_IRQ)) {
+ if (sums_need_wide_columns)
+ outp += sprintf(outp, "%s%8lld", (printed++ ? delim : ""), t->irq_count);
+ else
+ outp += sprintf(outp, "%s%lld", (printed++ ? delim : ""), t->irq_count);
+ }
/* SMI */
- if (do_smi)
- outp += sprintf(outp, "\t%d", t->smi_count);
-
- if (do_nhm_cstates)
- outp += sprintf(outp, "\t%.2f", 100.0 * t->c1/t->tsc);
-
- /* print per-core data only for 1st thread in core */
- if (!(t->flags & CPU_IS_FIRST_THREAD_IN_CORE))
- goto done;
-
- if (do_nhm_cstates && !do_slm_cstates && !do_knl_cstates)
- outp += sprintf(outp, "\t%.2f", 100.0 * c->c3/t->tsc);
- if (do_nhm_cstates)
- outp += sprintf(outp, "\t%.2f", 100.0 * c->c6/t->tsc);
- if (do_snb_cstates)
- outp += sprintf(outp, "\t%.2f", 100.0 * c->c7/t->tsc);
+ if (DO_BIC(BIC_SMI))
+ outp += sprintf(outp, "%s%d", (printed++ ? delim : ""), t->smi_count);
+ /* Added counters */
for (i = 0, mp = sys.tp; mp; i++, mp = mp->next) {
if (mp->format == FORMAT_RAW) {
if (mp->width == 32)
- outp += sprintf(outp, "\t0x%08lx", (unsigned long) t->counter[i]);
+ outp += sprintf(outp, "%s0x%08x", (printed++ ? delim : ""), (unsigned int) t->counter[i]);
else
- outp += sprintf(outp, "\t0x%016llx", t->counter[i]);
+ outp += sprintf(outp, "%s0x%016llx", (printed++ ? delim : ""), t->counter[i]);
} else if (mp->format == FORMAT_DELTA) {
- outp += sprintf(outp, "\t%8lld", t->counter[i]);
+ if ((mp->type == COUNTER_ITEMS) && sums_need_wide_columns)
+ outp += sprintf(outp, "%s%8lld", (printed++ ? delim : ""), t->counter[i]);
+ else
+ outp += sprintf(outp, "%s%lld", (printed++ ? delim : ""), t->counter[i]);
} else if (mp->format == FORMAT_PERCENT) {
- outp += sprintf(outp, "\t%.2f", 100.0 * t->counter[i]/t->tsc);
+ if (mp->type == COUNTER_USEC)
+ outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), t->counter[i]/interval_float/10000);
+ else
+ outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), 100.0 * t->counter[i]/tsc);
}
}
+ /* C1 */
+ if (DO_BIC(BIC_CPU_c1))
+ outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), 100.0 * t->c1/tsc);
- if (do_dts)
- outp += sprintf(outp, "\t%d", c->core_temp_c);
+
+ /* print per-core data only for 1st thread in core */
+ if (!(t->flags & CPU_IS_FIRST_THREAD_IN_CORE))
+ goto done;
+
+ if (DO_BIC(BIC_CPU_c3) && !do_slm_cstates && !do_knl_cstates)
+ outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), 100.0 * c->c3/tsc);
+ if (DO_BIC(BIC_CPU_c6))
+ outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), 100.0 * c->c6/tsc);
+ if (DO_BIC(BIC_CPU_c7))
+ outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), 100.0 * c->c7/tsc);
+
+ /* Mod%c6 */
+ if (DO_BIC(BIC_Mod_c6))
+ outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), 100.0 * c->mc6_us / tsc);
+
+ if (DO_BIC(BIC_CoreTmp))
+ outp += sprintf(outp, "%s%d", (printed++ ? delim : ""), c->core_temp_c);
for (i = 0, mp = sys.cp; mp; i++, mp = mp->next) {
if (mp->format == FORMAT_RAW) {
if (mp->width == 32)
- outp += sprintf(outp, "\t0x%08lx", (unsigned long) c->counter[i]);
+ outp += sprintf(outp, "%s0x%08x", (printed++ ? delim : ""), (unsigned int) c->counter[i]);
else
- outp += sprintf(outp, "\t0x%016llx", c->counter[i]);
+ outp += sprintf(outp, "%s0x%016llx", (printed++ ? delim : ""), c->counter[i]);
} else if (mp->format == FORMAT_DELTA) {
- outp += sprintf(outp, "\t%8lld", c->counter[i]);
+ if ((mp->type == COUNTER_ITEMS) && sums_need_wide_columns)
+ outp += sprintf(outp, "%s%8lld", (printed++ ? delim : ""), c->counter[i]);
+ else
+ outp += sprintf(outp, "%s%lld", (printed++ ? delim : ""), c->counter[i]);
} else if (mp->format == FORMAT_PERCENT) {
- outp += sprintf(outp, "\t%.2f", 100.0 * c->counter[i]/t->tsc);
+ outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), 100.0 * c->counter[i]/tsc);
}
}
goto done;
/* PkgTmp */
- if (do_ptm)
- outp += sprintf(outp, "\t%d", p->pkg_temp_c);
+ if (DO_BIC(BIC_PkgTmp))
+ outp += sprintf(outp, "%s%d", (printed++ ? delim : ""), p->pkg_temp_c);
/* GFXrc6 */
- if (do_gfx_rc6_ms) {
+ if (DO_BIC(BIC_GFX_rc6)) {
if (p->gfx_rc6_ms == -1) { /* detect GFX counter reset */
- outp += sprintf(outp, "\t**.**");
+ outp += sprintf(outp, "%s**.**", (printed++ ? delim : ""));
} else {
- outp += sprintf(outp, "\t%.2f",
+ outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""),
p->gfx_rc6_ms / 10.0 / interval_float);
}
}
/* GFXMHz */
- if (do_gfx_mhz)
- outp += sprintf(outp, "\t%d", p->gfx_mhz);
+ if (DO_BIC(BIC_GFXMHz))
+ outp += sprintf(outp, "%s%d", (printed++ ? delim : ""), p->gfx_mhz);
/* Totl%C0, Any%C0 GFX%C0 CPUGFX% */
if (do_skl_residency) {
- outp += sprintf(outp, "\t%.2f", 100.0 * p->pkg_wtd_core_c0/t->tsc);
- outp += sprintf(outp, "\t%.2f", 100.0 * p->pkg_any_core_c0/t->tsc);
- outp += sprintf(outp, "\t%.2f", 100.0 * p->pkg_any_gfxe_c0/t->tsc);
- outp += sprintf(outp, "\t%.2f", 100.0 * p->pkg_both_core_gfxe_c0/t->tsc);
- }
-
- if (do_pc2)
- outp += sprintf(outp, "\t%.2f", 100.0 * p->pc2/t->tsc);
- if (do_pc3)
- outp += sprintf(outp, "\t%.2f", 100.0 * p->pc3/t->tsc);
- if (do_pc6)
- outp += sprintf(outp, "\t%.2f", 100.0 * p->pc6/t->tsc);
- if (do_pc7)
- outp += sprintf(outp, "\t%.2f", 100.0 * p->pc7/t->tsc);
- if (do_c8_c9_c10) {
- outp += sprintf(outp, "\t%.2f", 100.0 * p->pc8/t->tsc);
- outp += sprintf(outp, "\t%.2f", 100.0 * p->pc9/t->tsc);
- outp += sprintf(outp, "\t%.2f", 100.0 * p->pc10/t->tsc);
+ outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), 100.0 * p->pkg_wtd_core_c0/tsc);
+ outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), 100.0 * p->pkg_any_core_c0/tsc);
+ outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), 100.0 * p->pkg_any_gfxe_c0/tsc);
+ outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), 100.0 * p->pkg_both_core_gfxe_c0/tsc);
}
+ if (DO_BIC(BIC_Pkgpc2))
+ outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), 100.0 * p->pc2/tsc);
+ if (DO_BIC(BIC_Pkgpc3))
+ outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), 100.0 * p->pc3/tsc);
+ if (DO_BIC(BIC_Pkgpc6))
+ outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), 100.0 * p->pc6/tsc);
+ if (DO_BIC(BIC_Pkgpc7))
+ outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), 100.0 * p->pc7/tsc);
+ if (DO_BIC(BIC_Pkgpc8))
+ outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), 100.0 * p->pc8/tsc);
+ if (DO_BIC(BIC_Pkgpc9))
+ outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), 100.0 * p->pc9/tsc);
+ if (DO_BIC(BIC_Pkgpc10))
+ outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), 100.0 * p->pc10/tsc);
+
/*
* If measurement interval exceeds minimum RAPL Joule Counter range,
* indicate that results are suspect by printing "**" in fraction place.
*/
if (interval_float < rapl_joule_counter_range)
- fmt8 = "\t%.2f";
+ fmt8 = "%s%.2f";
else
fmt8 = "%6.0f**";
- if (do_rapl && !rapl_joules) {
- if (do_rapl & RAPL_PKG)
- outp += sprintf(outp, fmt8, p->energy_pkg * rapl_energy_units / interval_float);
- if (do_rapl & RAPL_CORES_ENERGY_STATUS)
- outp += sprintf(outp, fmt8, p->energy_cores * rapl_energy_units / interval_float);
- if (do_rapl & RAPL_GFX)
- outp += sprintf(outp, fmt8, p->energy_gfx * rapl_energy_units / interval_float);
- if (do_rapl & RAPL_DRAM)
- outp += sprintf(outp, fmt8, p->energy_dram * rapl_dram_energy_units / interval_float);
- if (do_rapl & RAPL_PKG_PERF_STATUS)
- outp += sprintf(outp, fmt8, 100.0 * p->rapl_pkg_perf_status * rapl_time_units / interval_float);
- if (do_rapl & RAPL_DRAM_PERF_STATUS)
- outp += sprintf(outp, fmt8, 100.0 * p->rapl_dram_perf_status * rapl_time_units / interval_float);
- } else if (do_rapl && rapl_joules) {
- if (do_rapl & RAPL_PKG)
- outp += sprintf(outp, fmt8,
- p->energy_pkg * rapl_energy_units);
- if (do_rapl & RAPL_CORES)
- outp += sprintf(outp, fmt8,
- p->energy_cores * rapl_energy_units);
- if (do_rapl & RAPL_GFX)
- outp += sprintf(outp, fmt8,
- p->energy_gfx * rapl_energy_units);
- if (do_rapl & RAPL_DRAM)
- outp += sprintf(outp, fmt8,
- p->energy_dram * rapl_dram_energy_units);
- if (do_rapl & RAPL_PKG_PERF_STATUS)
- outp += sprintf(outp, fmt8, 100.0 * p->rapl_pkg_perf_status * rapl_time_units / interval_float);
- if (do_rapl & RAPL_DRAM_PERF_STATUS)
- outp += sprintf(outp, fmt8, 100.0 * p->rapl_dram_perf_status * rapl_time_units / interval_float);
- }
+ if (DO_BIC(BIC_PkgWatt))
+ outp += sprintf(outp, fmt8, (printed++ ? delim : ""), p->energy_pkg * rapl_energy_units / interval_float);
+ if (DO_BIC(BIC_CorWatt))
+ outp += sprintf(outp, fmt8, (printed++ ? delim : ""), p->energy_cores * rapl_energy_units / interval_float);
+ if (DO_BIC(BIC_GFXWatt))
+ outp += sprintf(outp, fmt8, (printed++ ? delim : ""), p->energy_gfx * rapl_energy_units / interval_float);
+ if (DO_BIC(BIC_RAMWatt))
+ outp += sprintf(outp, fmt8, (printed++ ? delim : ""), p->energy_dram * rapl_dram_energy_units / interval_float);
+ if (DO_BIC(BIC_Pkg_J))
+ outp += sprintf(outp, fmt8, (printed++ ? delim : ""), p->energy_pkg * rapl_energy_units);
+ if (DO_BIC(BIC_Cor_J))
+ outp += sprintf(outp, fmt8, (printed++ ? delim : ""), p->energy_cores * rapl_energy_units);
+ if (DO_BIC(BIC_GFX_J))
+ outp += sprintf(outp, fmt8, (printed++ ? delim : ""), p->energy_gfx * rapl_energy_units);
+ if (DO_BIC(BIC_RAM_J))
+ outp += sprintf(outp, fmt8, (printed++ ? delim : ""), p->energy_dram * rapl_dram_energy_units);
+ if (DO_BIC(BIC_PKG__))
+ outp += sprintf(outp, fmt8, (printed++ ? delim : ""), 100.0 * p->rapl_pkg_perf_status * rapl_time_units / interval_float);
+ if (DO_BIC(BIC_RAM__))
+ outp += sprintf(outp, fmt8, (printed++ ? delim : ""), 100.0 * p->rapl_dram_perf_status * rapl_time_units / interval_float);
+
for (i = 0, mp = sys.pp; mp; i++, mp = mp->next) {
if (mp->format == FORMAT_RAW) {
if (mp->width == 32)
- outp += sprintf(outp, "\t0x%08lx", (unsigned long) p->counter[i]);
+ outp += sprintf(outp, "%s0x%08x", (printed++ ? delim : ""), (unsigned int) p->counter[i]);
else
- outp += sprintf(outp, "\t0x%016llx", p->counter[i]);
+ outp += sprintf(outp, "%s0x%016llx", (printed++ ? delim : ""), p->counter[i]);
} else if (mp->format == FORMAT_DELTA) {
- outp += sprintf(outp, "\t%8lld", p->counter[i]);
+ if ((mp->type == COUNTER_ITEMS) && sums_need_wide_columns)
+ outp += sprintf(outp, "%s%8lld", (printed++ ? delim : ""), p->counter[i]);
+ else
+ outp += sprintf(outp, "%s%lld", (printed++ ? delim : ""), p->counter[i]);
} else if (mp->format == FORMAT_PERCENT) {
- outp += sprintf(outp, "\t%.2f", 100.0 * p->counter[i]/t->tsc);
+ outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), 100.0 * p->counter[i]/tsc);
}
}
static int printed;
if (!printed || !summary_only)
- print_header();
+ print_header("\t");
if (topo.num_cpus > 1)
format_counters(&average.threads, &average.cores,
old->pkg_both_core_gfxe_c0 = new->pkg_both_core_gfxe_c0 - old->pkg_both_core_gfxe_c0;
}
old->pc2 = new->pc2 - old->pc2;
- if (do_pc3)
+ if (DO_BIC(BIC_Pkgpc3))
old->pc3 = new->pc3 - old->pc3;
- if (do_pc6)
+ if (DO_BIC(BIC_Pkgpc6))
old->pc6 = new->pc6 - old->pc6;
- if (do_pc7)
+ if (DO_BIC(BIC_Pkgpc7))
old->pc7 = new->pc7 - old->pc7;
old->pc8 = new->pc8 - old->pc8;
old->pc9 = new->pc9 - old->pc9;
old->c6 = new->c6 - old->c6;
old->c7 = new->c7 - old->c7;
old->core_temp_c = new->core_temp_c;
+ old->mc6_us = new->mc6_us - old->mc6_us;
for (i = 0, mp = sys.cp; mp; i++, mp = mp->next) {
if (mp->format == FORMAT_RAW)
old->c1 = new->c1 - old->c1;
- if (has_aperf) {
+ if (DO_BIC(BIC_Avg_MHz) || DO_BIC(BIC_Busy) || DO_BIC(BIC_Bzy_MHz)) {
if ((new->aperf > old->aperf) && (new->mperf > old->mperf)) {
old->aperf = new->aperf - old->aperf;
old->mperf = new->mperf - old->mperf;
old->c1 = 0;
else {
/* normal case, derive c1 */
- old->c1 = old->tsc - old->mperf - core_delta->c3
+ old->c1 = (old->tsc * tsc_tweak) - old->mperf - core_delta->c3
- core_delta->c6 - core_delta->c7;
}
}
old->mperf = 1; /* divide by 0 protection */
}
- if (do_irq)
+ if (DO_BIC(BIC_IRQ))
old->irq_count = new->irq_count - old->irq_count;
- if (do_smi)
+ if (DO_BIC(BIC_SMI))
old->smi_count = new->smi_count - old->smi_count;
for (i = 0, mp = sys.tp; mp; i++, mp = mp->next) {
c->c3 = 0;
c->c6 = 0;
c->c7 = 0;
+ c->mc6_us = 0;
c->core_temp_c = 0;
p->pkg_wtd_core_c0 = 0;
p->pkg_both_core_gfxe_c0 = 0;
p->pc2 = 0;
- if (do_pc3)
+ if (DO_BIC(BIC_Pkgpc3))
p->pc3 = 0;
- if (do_pc6)
+ if (DO_BIC(BIC_Pkgpc6))
p->pc6 = 0;
- if (do_pc7)
+ if (DO_BIC(BIC_Pkgpc7))
p->pc7 = 0;
p->pc8 = 0;
p->pc9 = 0;
p->gfx_rc6_ms = 0;
p->gfx_mhz = 0;
-
for (i = 0, mp = sys.tp; mp; i++, mp = mp->next)
t->counter[i] = 0;
average.cores.c3 += c->c3;
average.cores.c6 += c->c6;
average.cores.c7 += c->c7;
+ average.cores.mc6_us += c->mc6_us;
average.cores.core_temp_c = MAX(average.cores.core_temp_c, c->core_temp_c);
}
average.packages.pc2 += p->pc2;
- if (do_pc3)
+ if (DO_BIC(BIC_Pkgpc3))
average.packages.pc3 += p->pc3;
- if (do_pc6)
+ if (DO_BIC(BIC_Pkgpc6))
average.packages.pc6 += p->pc6;
- if (do_pc7)
+ if (DO_BIC(BIC_Pkgpc7))
average.packages.pc7 += p->pc7;
average.packages.pc8 += p->pc8;
average.packages.pc9 += p->pc9;
average.threads.mperf /= topo.num_cpus;
average.threads.c1 /= topo.num_cpus;
+ if (average.threads.irq_count > 9999999)
+ sums_need_wide_columns = 1;
+
average.cores.c3 /= topo.num_cores;
average.cores.c6 /= topo.num_cores;
average.cores.c7 /= topo.num_cores;
+ average.cores.mc6_us /= topo.num_cores;
if (do_skl_residency) {
average.packages.pkg_wtd_core_c0 /= topo.num_packages;
}
average.packages.pc2 /= topo.num_packages;
- if (do_pc3)
+ if (DO_BIC(BIC_Pkgpc3))
average.packages.pc3 /= topo.num_packages;
- if (do_pc6)
+ if (DO_BIC(BIC_Pkgpc6))
average.packages.pc6 /= topo.num_packages;
- if (do_pc7)
+ if (DO_BIC(BIC_Pkgpc7))
average.packages.pc7 /= topo.num_packages;
average.packages.pc8 /= topo.num_packages;
for (i = 0, mp = sys.tp; mp; i++, mp = mp->next) {
if (mp->format == FORMAT_RAW)
continue;
+ if (mp->type == COUNTER_ITEMS) {
+ if (average.threads.counter[i] > 9999999)
+ sums_need_wide_columns = 1;
+ continue;
+ }
average.threads.counter[i] /= topo.num_cpus;
}
for (i = 0, mp = sys.cp; mp; i++, mp = mp->next) {
if (mp->format == FORMAT_RAW)
continue;
+ if (mp->type == COUNTER_ITEMS) {
+ if (average.cores.counter[i] > 9999999)
+ sums_need_wide_columns = 1;
+ }
average.cores.counter[i] /= topo.num_cores;
}
for (i = 0, mp = sys.pp; mp; i++, mp = mp->next) {
if (mp->format == FORMAT_RAW)
continue;
+ if (mp->type == COUNTER_ITEMS) {
+ if (average.packages.counter[i] > 9999999)
+ sums_need_wide_columns = 1;
+ }
average.packages.counter[i] /= topo.num_packages;
}
}
return low | ((unsigned long long)high) << 32;
}
+/*
+ * Open a file, and exit on failure
+ */
+FILE *fopen_or_die(const char *path, const char *mode)
+{
+ FILE *filep = fopen(path, mode);
+
+ if (!filep)
+ err(1, "%s: open failed", path);
+ return filep;
+}
+/*
+ * snapshot_sysfs_counter()
+ *
+ * return snapshot of given counter
+ */
+unsigned long long snapshot_sysfs_counter(char *path)
+{
+ FILE *fp;
+ int retval;
+ unsigned long long counter;
+
+ fp = fopen_or_die(path, "r");
+
+ retval = fscanf(fp, "%lld", &counter);
+ if (retval != 1)
+ err(1, "snapshot_sysfs_counter(%s)", path);
+
+ fclose(fp);
+
+ return counter;
+}
+
+int get_mp(int cpu, struct msr_counter *mp, unsigned long long *counterp)
+{
+ if (mp->msr_num != 0) {
+ if (get_msr(cpu, mp->msr_num, counterp))
+ return -1;
+ } else {
+ char path[128];
+
+ if (mp->flags & SYSFS_PERCPU) {
+ sprintf(path, "/sys/devices/system/cpu/cpu%d/%s",
+ cpu, mp->path);
+
+ *counterp = snapshot_sysfs_counter(path);
+ } else {
+ *counterp = snapshot_sysfs_counter(mp->path);
+ }
+ }
+
+ return 0;
+}
+
/*
* get_counters(...)
* migrate to cpu
retry:
t->tsc = rdtsc(); /* we are running on local CPU of interest */
- if (has_aperf) {
+ if (DO_BIC(BIC_Avg_MHz) || DO_BIC(BIC_Busy) || DO_BIC(BIC_Bzy_MHz)) {
unsigned long long tsc_before, tsc_between, tsc_after, aperf_time, mperf_time;
/*
t->mperf = t->mperf * aperf_mperf_multiplier;
}
- if (do_irq)
+ if (DO_BIC(BIC_IRQ))
t->irq_count = irqs_per_cpu[cpu];
- if (do_smi) {
+ if (DO_BIC(BIC_SMI)) {
if (get_msr(cpu, MSR_SMI_COUNT, &msr))
return -5;
t->smi_count = msr & 0xFFFFFFFF;
}
-
- if (use_c1_residency_msr) {
+ if (DO_BIC(BIC_CPU_c1) && use_c1_residency_msr) {
if (get_msr(cpu, MSR_CORE_C1_RES, &t->c1))
return -6;
}
for (i = 0, mp = sys.tp; mp; i++, mp = mp->next) {
- if (get_msr(cpu, mp->msr_num, &t->counter[i]))
+ if (get_mp(cpu, mp, &t->counter[i]))
return -10;
}
-
/* collect core counters only for 1st thread in core */
if (!(t->flags & CPU_IS_FIRST_THREAD_IN_CORE))
return 0;
- if (do_nhm_cstates && !do_slm_cstates && !do_knl_cstates) {
+ if (DO_BIC(BIC_CPU_c3) && !do_slm_cstates && !do_knl_cstates) {
if (get_msr(cpu, MSR_CORE_C3_RESIDENCY, &c->c3))
return -6;
}
- if (do_nhm_cstates && !do_knl_cstates) {
+ if (DO_BIC(BIC_CPU_c6) && !do_knl_cstates) {
if (get_msr(cpu, MSR_CORE_C6_RESIDENCY, &c->c6))
return -7;
} else if (do_knl_cstates) {
return -7;
}
- if (do_snb_cstates)
+ if (DO_BIC(BIC_CPU_c7))
if (get_msr(cpu, MSR_CORE_C7_RESIDENCY, &c->c7))
return -8;
- if (do_dts) {
+ if (DO_BIC(BIC_Mod_c6))
+ if (get_msr(cpu, MSR_MODULE_C6_RES_MS, &c->mc6_us))
+ return -8;
+
+ if (DO_BIC(BIC_CoreTmp)) {
if (get_msr(cpu, MSR_IA32_THERM_STATUS, &msr))
return -9;
c->core_temp_c = tcc_activation_temp - ((msr >> 16) & 0x7F);
}
for (i = 0, mp = sys.cp; mp; i++, mp = mp->next) {
- if (get_msr(cpu, mp->msr_num, &c->counter[i]))
+ if (get_mp(cpu, mp, &c->counter[i]))
return -10;
}
if (get_msr(cpu, MSR_PKG_BOTH_CORE_GFXE_C0_RES, &p->pkg_both_core_gfxe_c0))
return -13;
}
- if (do_pc3)
+ if (DO_BIC(BIC_Pkgpc3))
if (get_msr(cpu, MSR_PKG_C3_RESIDENCY, &p->pc3))
return -9;
- if (do_pc6)
- if (get_msr(cpu, MSR_PKG_C6_RESIDENCY, &p->pc6))
- return -10;
- if (do_pc2)
+ if (DO_BIC(BIC_Pkgpc6)) {
+ if (do_slm_cstates) {
+ if (get_msr(cpu, MSR_ATOM_PKG_C6_RESIDENCY, &p->pc6))
+ return -10;
+ } else {
+ if (get_msr(cpu, MSR_PKG_C6_RESIDENCY, &p->pc6))
+ return -10;
+ }
+ }
+
+ if (DO_BIC(BIC_Pkgpc2))
if (get_msr(cpu, MSR_PKG_C2_RESIDENCY, &p->pc2))
return -11;
- if (do_pc7)
+ if (DO_BIC(BIC_Pkgpc7))
if (get_msr(cpu, MSR_PKG_C7_RESIDENCY, &p->pc7))
return -12;
- if (do_c8_c9_c10) {
+ if (DO_BIC(BIC_Pkgpc8))
if (get_msr(cpu, MSR_PKG_C8_RESIDENCY, &p->pc8))
return -13;
+ if (DO_BIC(BIC_Pkgpc9))
if (get_msr(cpu, MSR_PKG_C9_RESIDENCY, &p->pc9))
return -13;
+ if (DO_BIC(BIC_Pkgpc10))
if (get_msr(cpu, MSR_PKG_C10_RESIDENCY, &p->pc10))
return -13;
- }
+
if (do_rapl & RAPL_PKG) {
if (get_msr(cpu, MSR_PKG_ENERGY_STATUS, &msr))
return -13;
return -16;
p->rapl_dram_perf_status = msr & 0xFFFFFFFF;
}
- if (do_ptm) {
+ if (DO_BIC(BIC_PkgTmp)) {
if (get_msr(cpu, MSR_IA32_PACKAGE_THERM_STATUS, &msr))
return -17;
p->pkg_temp_c = tcc_activation_temp - ((msr >> 16) & 0x7F);
}
- if (do_gfx_rc6_ms)
+ if (DO_BIC(BIC_GFX_rc6))
p->gfx_rc6_ms = gfx_cur_rc6_ms;
- if (do_gfx_mhz)
+ if (DO_BIC(BIC_GFXMHz))
p->gfx_mhz = gfx_cur_mhz;
for (i = 0, mp = sys.pp; mp; i++, mp = mp->next) {
- if (get_msr(cpu, mp->msr_num, &p->counter[i]))
+ if (get_mp(cpu, mp, &p->counter[i]))
return -10;
}
int nhm_pkg_cstate_limits[16] = {PCL__0, PCL__1, PCL__3, PCL__6, PCL__7, PCLRSV, PCLRSV, PCLUNL, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLRSV};
int snb_pkg_cstate_limits[16] = {PCL__0, PCL__2, PCL_6N, PCL_6R, PCL__7, PCL_7S, PCLRSV, PCLUNL, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLRSV};
int hsw_pkg_cstate_limits[16] = {PCL__0, PCL__2, PCL__3, PCL__6, PCL__7, PCL_7S, PCL__8, PCL__9, PCLUNL, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLRSV};
-int slv_pkg_cstate_limits[16] = {PCL__0, PCL__1, PCLRSV, PCLRSV, PCL__4, PCLRSV, PCL__6, PCL__7, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLRSV};
-int amt_pkg_cstate_limits[16] = {PCL__0, PCL__1, PCL__2, PCLRSV, PCLRSV, PCLRSV, PCL__6, PCL__7, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLRSV};
+int slv_pkg_cstate_limits[16] = {PCL__0, PCL__1, PCLRSV, PCLRSV, PCL__4, PCLRSV, PCL__6, PCL__7, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCL__6, PCL__7};
+int amt_pkg_cstate_limits[16] = {PCLUNL, PCL__1, PCL__2, PCLRSV, PCLRSV, PCLRSV, PCL__6, PCL__7, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLRSV};
int phi_pkg_cstate_limits[16] = {PCL__0, PCL__2, PCL_6N, PCL_6R, PCLRSV, PCLRSV, PCLRSV, PCLUNL, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLRSV};
int bxt_pkg_cstate_limits[16] = {PCL__0, PCL__2, PCLUNL, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLRSV};
int skx_pkg_cstate_limits[16] = {PCL__0, PCL__2, PCL_6N, PCL_6R, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLUNL, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLRSV};
fprintf(outf, "cpu%d: MSR_PLATFORM_INFO: 0x%08llx\n", base_cpu, msr);
ratio = (msr >> 40) & 0xFF;
- fprintf(outf, "%d * %.0f = %.0f MHz max efficiency frequency\n",
+ fprintf(outf, "%d * %.1f = %.1f MHz max efficiency frequency\n",
ratio, bclk, ratio * bclk);
ratio = (msr >> 8) & 0xFF;
- fprintf(outf, "%d * %.0f = %.0f MHz base frequency\n",
+ fprintf(outf, "%d * %.1f = %.1f MHz base frequency\n",
ratio, bclk, ratio * bclk);
get_msr(base_cpu, MSR_IA32_POWER_CTL, &msr);
ratio = (msr >> 8) & 0xFF;
if (ratio)
- fprintf(outf, "%d * %.0f = %.0f MHz max turbo 18 active cores\n",
+ fprintf(outf, "%d * %.1f = %.1f MHz max turbo 18 active cores\n",
ratio, bclk, ratio * bclk);
ratio = (msr >> 0) & 0xFF;
if (ratio)
- fprintf(outf, "%d * %.0f = %.0f MHz max turbo 17 active cores\n",
+ fprintf(outf, "%d * %.1f = %.1f MHz max turbo 17 active cores\n",
ratio, bclk, ratio * bclk);
return;
}
ratio = (msr >> 56) & 0xFF;
if (ratio)
- fprintf(outf, "%d * %.0f = %.0f MHz max turbo 16 active cores\n",
+ fprintf(outf, "%d * %.1f = %.1f MHz max turbo 16 active cores\n",
ratio, bclk, ratio * bclk);
ratio = (msr >> 48) & 0xFF;
if (ratio)
- fprintf(outf, "%d * %.0f = %.0f MHz max turbo 15 active cores\n",
+ fprintf(outf, "%d * %.1f = %.1f MHz max turbo 15 active cores\n",
ratio, bclk, ratio * bclk);
ratio = (msr >> 40) & 0xFF;
if (ratio)
- fprintf(outf, "%d * %.0f = %.0f MHz max turbo 14 active cores\n",
+ fprintf(outf, "%d * %.1f = %.1f MHz max turbo 14 active cores\n",
ratio, bclk, ratio * bclk);
ratio = (msr >> 32) & 0xFF;
if (ratio)
- fprintf(outf, "%d * %.0f = %.0f MHz max turbo 13 active cores\n",
+ fprintf(outf, "%d * %.1f = %.1f MHz max turbo 13 active cores\n",
ratio, bclk, ratio * bclk);
ratio = (msr >> 24) & 0xFF;
if (ratio)
- fprintf(outf, "%d * %.0f = %.0f MHz max turbo 12 active cores\n",
+ fprintf(outf, "%d * %.1f = %.1f MHz max turbo 12 active cores\n",
ratio, bclk, ratio * bclk);
ratio = (msr >> 16) & 0xFF;
if (ratio)
- fprintf(outf, "%d * %.0f = %.0f MHz max turbo 11 active cores\n",
+ fprintf(outf, "%d * %.1f = %.1f MHz max turbo 11 active cores\n",
ratio, bclk, ratio * bclk);
ratio = (msr >> 8) & 0xFF;
if (ratio)
- fprintf(outf, "%d * %.0f = %.0f MHz max turbo 10 active cores\n",
+ fprintf(outf, "%d * %.1f = %.1f MHz max turbo 10 active cores\n",
ratio, bclk, ratio * bclk);
ratio = (msr >> 0) & 0xFF;
if (ratio)
- fprintf(outf, "%d * %.0f = %.0f MHz max turbo 9 active cores\n",
+ fprintf(outf, "%d * %.1f = %.1f MHz max turbo 9 active cores\n",
ratio, bclk, ratio * bclk);
return;
}
+int has_turbo_ratio_group_limits(int family, int model)
+{
+
+ if (!genuine_intel)
+ return 0;
+
+ switch (model) {
+ case INTEL_FAM6_ATOM_GOLDMONT:
+ case INTEL_FAM6_SKYLAKE_X:
+ case INTEL_FAM6_ATOM_DENVERTON:
+ return 1;
+ }
+ return 0;
+}
static void
-dump_nhm_turbo_ratio_limits(void)
+dump_turbo_ratio_limits(int family, int model)
{
- unsigned long long msr;
- unsigned int ratio;
+ unsigned long long msr, core_counts;
+ unsigned int ratio, group_size;
get_msr(base_cpu, MSR_TURBO_RATIO_LIMIT, &msr);
-
fprintf(outf, "cpu%d: MSR_TURBO_RATIO_LIMIT: 0x%08llx\n", base_cpu, msr);
+ if (has_turbo_ratio_group_limits(family, model)) {
+ get_msr(base_cpu, MSR_TURBO_RATIO_LIMIT1, &core_counts);
+ fprintf(outf, "cpu%d: MSR_TURBO_RATIO_LIMIT1: 0x%08llx\n", base_cpu, core_counts);
+ } else {
+ core_counts = 0x0807060504030201;
+ }
+
ratio = (msr >> 56) & 0xFF;
+ group_size = (core_counts >> 56) & 0xFF;
if (ratio)
- fprintf(outf, "%d * %.0f = %.0f MHz max turbo 8 active cores\n",
- ratio, bclk, ratio * bclk);
+ fprintf(outf, "%d * %.1f = %.1f MHz max turbo %d active cores\n",
+ ratio, bclk, ratio * bclk, group_size);
ratio = (msr >> 48) & 0xFF;
+ group_size = (core_counts >> 48) & 0xFF;
if (ratio)
- fprintf(outf, "%d * %.0f = %.0f MHz max turbo 7 active cores\n",
- ratio, bclk, ratio * bclk);
+ fprintf(outf, "%d * %.1f = %.1f MHz max turbo %d active cores\n",
+ ratio, bclk, ratio * bclk, group_size);
ratio = (msr >> 40) & 0xFF;
+ group_size = (core_counts >> 40) & 0xFF;
if (ratio)
- fprintf(outf, "%d * %.0f = %.0f MHz max turbo 6 active cores\n",
- ratio, bclk, ratio * bclk);
+ fprintf(outf, "%d * %.1f = %.1f MHz max turbo %d active cores\n",
+ ratio, bclk, ratio * bclk, group_size);
ratio = (msr >> 32) & 0xFF;
+ group_size = (core_counts >> 32) & 0xFF;
if (ratio)
- fprintf(outf, "%d * %.0f = %.0f MHz max turbo 5 active cores\n",
- ratio, bclk, ratio * bclk);
+ fprintf(outf, "%d * %.1f = %.1f MHz max turbo %d active cores\n",
+ ratio, bclk, ratio * bclk, group_size);
ratio = (msr >> 24) & 0xFF;
+ group_size = (core_counts >> 24) & 0xFF;
if (ratio)
- fprintf(outf, "%d * %.0f = %.0f MHz max turbo 4 active cores\n",
- ratio, bclk, ratio * bclk);
+ fprintf(outf, "%d * %.1f = %.1f MHz max turbo %d active cores\n",
+ ratio, bclk, ratio * bclk, group_size);
ratio = (msr >> 16) & 0xFF;
+ group_size = (core_counts >> 16) & 0xFF;
if (ratio)
- fprintf(outf, "%d * %.0f = %.0f MHz max turbo 3 active cores\n",
- ratio, bclk, ratio * bclk);
+ fprintf(outf, "%d * %.1f = %.1f MHz max turbo %d active cores\n",
+ ratio, bclk, ratio * bclk, group_size);
ratio = (msr >> 8) & 0xFF;
+ group_size = (core_counts >> 8) & 0xFF;
if (ratio)
- fprintf(outf, "%d * %.0f = %.0f MHz max turbo 2 active cores\n",
- ratio, bclk, ratio * bclk);
+ fprintf(outf, "%d * %.1f = %.1f MHz max turbo %d active cores\n",
+ ratio, bclk, ratio * bclk, group_size);
ratio = (msr >> 0) & 0xFF;
+ group_size = (core_counts >> 0) & 0xFF;
if (ratio)
- fprintf(outf, "%d * %.0f = %.0f MHz max turbo 1 active cores\n",
- ratio, bclk, ratio * bclk);
+ fprintf(outf, "%d * %.1f = %.1f MHz max turbo %d active cores\n",
+ ratio, bclk, ratio * bclk, group_size);
return;
}
+static void
+dump_atom_turbo_ratio_limits(void)
+{
+ unsigned long long msr;
+ unsigned int ratio;
+
+ get_msr(base_cpu, MSR_ATOM_CORE_RATIOS, &msr);
+ fprintf(outf, "cpu%d: MSR_ATOM_CORE_RATIOS: 0x%08llx\n", base_cpu, msr & 0xFFFFFFFF);
+
+ ratio = (msr >> 0) & 0x3F;
+ if (ratio)
+ fprintf(outf, "%d * %.1f = %.1f MHz minimum operating frequency\n",
+ ratio, bclk, ratio * bclk);
+
+ ratio = (msr >> 8) & 0x3F;
+ if (ratio)
+ fprintf(outf, "%d * %.1f = %.1f MHz low frequency mode (LFM)\n",
+ ratio, bclk, ratio * bclk);
+
+ ratio = (msr >> 16) & 0x3F;
+ if (ratio)
+ fprintf(outf, "%d * %.1f = %.1f MHz base frequency\n",
+ ratio, bclk, ratio * bclk);
+
+ get_msr(base_cpu, MSR_ATOM_CORE_TURBO_RATIOS, &msr);
+ fprintf(outf, "cpu%d: MSR_ATOM_CORE_TURBO_RATIOS: 0x%08llx\n", base_cpu, msr & 0xFFFFFFFF);
+
+ ratio = (msr >> 24) & 0x3F;
+ if (ratio)
+ fprintf(outf, "%d * %.1f = %.1f MHz max turbo 4 active cores\n",
+ ratio, bclk, ratio * bclk);
+
+ ratio = (msr >> 16) & 0x3F;
+ if (ratio)
+ fprintf(outf, "%d * %.1f = %.1f MHz max turbo 3 active cores\n",
+ ratio, bclk, ratio * bclk);
+
+ ratio = (msr >> 8) & 0x3F;
+ if (ratio)
+ fprintf(outf, "%d * %.1f = %.1f MHz max turbo 2 active cores\n",
+ ratio, bclk, ratio * bclk);
+
+ ratio = (msr >> 0) & 0x3F;
+ if (ratio)
+ fprintf(outf, "%d * %.1f = %.1f MHz max turbo 1 active core\n",
+ ratio, bclk, ratio * bclk);
+}
+
static void
dump_knl_turbo_ratio_limits(void)
{
for (i = buckets_no - 1; i >= 0; i--)
if (i > 0 ? ratio[i] != ratio[i - 1] : 1)
fprintf(outf,
- "%d * %.0f = %.0f MHz max turbo %d active cores\n",
+ "%d * %.1f = %.1f MHz max turbo %d active cores\n",
ratio[i], bclk, ratio[i] * bclk, cores[i]);
}
{
unsigned long long msr;
- get_msr(base_cpu, MSR_NHM_SNB_PKG_CST_CFG_CTL, &msr);
+ get_msr(base_cpu, MSR_PKG_CST_CONFIG_CONTROL, &msr);
#define SNB_C1_AUTO_UNDEMOTE (1UL << 27)
#define SNB_C3_AUTO_UNDEMOTE (1UL << 28)
- fprintf(outf, "cpu%d: MSR_NHM_SNB_PKG_CST_CFG_CTL: 0x%08llx", base_cpu, msr);
+ fprintf(outf, "cpu%d: MSR_PKG_CST_CONFIG_CONTROL: 0x%08llx", base_cpu, msr);
fprintf(outf, " (%s%s%s%s%slocked: pkg-cstate-limit=%d: %s)\n",
(msr & SNB_C3_AUTO_UNDEMOTE) ? "UNdemote-C3, " : "",
free(irqs_per_cpu);
}
-/*
- * Open a file, and exit on failure
- */
-FILE *fopen_or_die(const char *path, const char *mode)
-{
- FILE *filep = fopen(path, mode);
- if (!filep)
- err(1, "%s: open failed", path);
- return filep;
-}
/*
* Parse a file containing a single int.
*/
int snapshot_proc_sysfs_files(void)
{
- if (snapshot_proc_interrupts())
- return 1;
+ if (DO_BIC(BIC_IRQ))
+ if (snapshot_proc_interrupts())
+ return 1;
- if (do_gfx_rc6_ms)
+ if (DO_BIC(BIC_GFX_rc6))
snapshot_gfx_rc6_ms();
- if (do_gfx_mhz)
+ if (DO_BIC(BIC_GFXMHz))
snapshot_gfx_mhz();
return 0;
* MSR_SMI_COUNT 0x00000034
*
* MSR_PLATFORM_INFO 0x000000ce
- * MSR_NHM_SNB_PKG_CST_CFG_CTL 0x000000e2
+ * MSR_PKG_CST_CONFIG_CONTROL 0x000000e2
+ *
+ * MSR_MISC_PWR_MGMT 0x000001aa
*
* MSR_PKG_C3_RESIDENCY 0x000003f8
* MSR_PKG_C6_RESIDENCY 0x000003f9
* MSR_CORE_C6_RESIDENCY 0x000003fd
*
* Side effect:
- * sets global pkg_cstate_limit to decode MSR_NHM_SNB_PKG_CST_CFG_CTL
+ * sets global pkg_cstate_limit to decode MSR_PKG_CST_CONFIG_CONTROL
+ * sets has_misc_feature_control
*/
int probe_nhm_msrs(unsigned int family, unsigned int model)
{
case INTEL_FAM6_IVYBRIDGE: /* IVB */
case INTEL_FAM6_IVYBRIDGE_X: /* IVB Xeon */
pkg_cstate_limits = snb_pkg_cstate_limits;
+ has_misc_feature_control = 1;
break;
case INTEL_FAM6_HASWELL_CORE: /* HSW */
case INTEL_FAM6_HASWELL_X: /* HSX */
case INTEL_FAM6_KABYLAKE_MOBILE: /* KBL */
case INTEL_FAM6_KABYLAKE_DESKTOP: /* KBL */
pkg_cstate_limits = hsw_pkg_cstate_limits;
+ has_misc_feature_control = 1;
break;
case INTEL_FAM6_SKYLAKE_X: /* SKX */
pkg_cstate_limits = skx_pkg_cstate_limits;
+ has_misc_feature_control = 1;
break;
case INTEL_FAM6_ATOM_SILVERMONT1: /* BYT */
+ no_MSR_MISC_PWR_MGMT = 1;
case INTEL_FAM6_ATOM_SILVERMONT2: /* AVN */
pkg_cstate_limits = slv_pkg_cstate_limits;
break;
case INTEL_FAM6_ATOM_AIRMONT: /* AMT */
pkg_cstate_limits = amt_pkg_cstate_limits;
+ no_MSR_MISC_PWR_MGMT = 1;
break;
case INTEL_FAM6_XEON_PHI_KNL: /* PHI */
case INTEL_FAM6_XEON_PHI_KNM:
pkg_cstate_limits = phi_pkg_cstate_limits;
break;
case INTEL_FAM6_ATOM_GOLDMONT: /* BXT */
+ case INTEL_FAM6_ATOM_GEMINI_LAKE:
case INTEL_FAM6_ATOM_DENVERTON: /* DNV */
pkg_cstate_limits = bxt_pkg_cstate_limits;
break;
default:
return 0;
}
- get_msr(base_cpu, MSR_NHM_SNB_PKG_CST_CFG_CTL, &msr);
+ get_msr(base_cpu, MSR_PKG_CST_CONFIG_CONTROL, &msr);
pkg_cstate_limit = pkg_cstate_limits[msr & 0xF];
get_msr(base_cpu, MSR_PLATFORM_INFO, &msr);
has_base_hz = 1;
return 1;
}
-int has_nhm_turbo_ratio_limit(unsigned int family, unsigned int model)
+/*
+ * SLV client has support for unique MSRs:
+ *
+ * MSR_CC6_DEMOTION_POLICY_CONFIG
+ * MSR_MC6_DEMOTION_POLICY_CONFIG
+ */
+
+int has_slv_msrs(unsigned int family, unsigned int model)
{
+ if (!genuine_intel)
+ return 0;
+
+ switch (model) {
+ case INTEL_FAM6_ATOM_SILVERMONT1:
+ case INTEL_FAM6_ATOM_MERRIFIELD:
+ case INTEL_FAM6_ATOM_MOOREFIELD:
+ return 1;
+ }
+ return 0;
+}
+int is_dnv(unsigned int family, unsigned int model)
+{
+
+ if (!genuine_intel)
+ return 0;
+
+ switch (model) {
+ case INTEL_FAM6_ATOM_DENVERTON:
+ return 1;
+ }
+ return 0;
+}
+int is_bdx(unsigned int family, unsigned int model)
+{
+
+ if (!genuine_intel)
+ return 0;
+
+ switch (model) {
+ case INTEL_FAM6_BROADWELL_X:
+ case INTEL_FAM6_BROADWELL_XEON_D:
+ return 1;
+ }
+ return 0;
+}
+int is_skx(unsigned int family, unsigned int model)
+{
+
+ if (!genuine_intel)
+ return 0;
+
+ switch (model) {
+ case INTEL_FAM6_SKYLAKE_X:
+ return 1;
+ }
+ return 0;
+}
+
+int has_turbo_ratio_limit(unsigned int family, unsigned int model)
+{
+ if (has_slv_msrs(family, model))
+ return 0;
+
switch (model) {
/* Nehalem compatible, but do not include turbo-ratio limit support */
case INTEL_FAM6_NEHALEM_EX: /* Nehalem-EX Xeon - Beckton */
return 1;
}
}
+int has_atom_turbo_ratio_limit(unsigned int family, unsigned int model)
+{
+ if (has_slv_msrs(family, model))
+ return 1;
+
+ return 0;
+}
int has_ivt_turbo_ratio_limit(unsigned int family, unsigned int model)
{
if (!genuine_intel)
return 0;
}
}
+int has_glm_turbo_ratio_limit(unsigned int family, unsigned int model)
+{
+ if (!genuine_intel)
+ return 0;
+
+ if (family != 6)
+ return 0;
+
+ switch (model) {
+ case INTEL_FAM6_ATOM_GOLDMONT:
+ case INTEL_FAM6_SKYLAKE_X:
+ return 1;
+ default:
+ return 0;
+ }
+}
int has_config_tdp(unsigned int family, unsigned int model)
{
if (!genuine_intel)
if (has_ivt_turbo_ratio_limit(family, model))
dump_ivt_turbo_ratio_limits();
- if (has_nhm_turbo_ratio_limit(family, model))
- dump_nhm_turbo_ratio_limits();
+ if (has_turbo_ratio_limit(family, model))
+ dump_turbo_ratio_limits(family, model);
+
+ if (has_atom_turbo_ratio_limit(family, model))
+ dump_atom_turbo_ratio_limits();
if (has_knl_turbo_ratio_limit(family, model))
dump_knl_turbo_ratio_limits();
dump_nhm_cst_cfg();
}
+static void
+dump_sysfs_cstate_config(void)
+{
+ char path[64];
+ char name_buf[16];
+ char desc[64];
+ FILE *input;
+ int state;
+ char *sp;
+
+ if (!DO_BIC(BIC_sysfs))
+ return;
+
+ for (state = 0; state < 10; ++state) {
+
+ sprintf(path, "/sys/devices/system/cpu/cpu%d/cpuidle/state%d/name",
+ base_cpu, state);
+ input = fopen(path, "r");
+ if (input == NULL)
+ continue;
+ fgets(name_buf, sizeof(name_buf), input);
+
+ /* truncate "C1-HSW\n" to "C1", or truncate "C1\n" to "C1" */
+ sp = strchr(name_buf, '-');
+ if (!sp)
+ sp = strchrnul(name_buf, '\n');
+ *sp = '\0';
+
+ fclose(input);
+
+ sprintf(path, "/sys/devices/system/cpu/cpu%d/cpuidle/state%d/desc",
+ base_cpu, state);
+ input = fopen(path, "r");
+ if (input == NULL)
+ continue;
+ fgets(desc, sizeof(desc), input);
+
+ fprintf(outf, "cpu%d: %s: %s", base_cpu, name_buf, desc);
+ fclose(input);
+ }
+}
+static void
+dump_sysfs_pstate_config(void)
+{
+ char path[64];
+ char driver_buf[64];
+ char governor_buf[64];
+ FILE *input;
+ int turbo;
+
+ sprintf(path, "/sys/devices/system/cpu/cpu%d/cpufreq/scaling_driver",
+ base_cpu);
+ input = fopen(path, "r");
+ if (input == NULL) {
+ fprintf(stderr, "NSFOD %s\n", path);
+ return;
+ }
+ fgets(driver_buf, sizeof(driver_buf), input);
+ fclose(input);
+
+ sprintf(path, "/sys/devices/system/cpu/cpu%d/cpufreq/scaling_governor",
+ base_cpu);
+ input = fopen(path, "r");
+ if (input == NULL) {
+ fprintf(stderr, "NSFOD %s\n", path);
+ return;
+ }
+ fgets(governor_buf, sizeof(governor_buf), input);
+ fclose(input);
+
+ fprintf(outf, "cpu%d: cpufreq driver: %s", base_cpu, driver_buf);
+ fprintf(outf, "cpu%d: cpufreq governor: %s", base_cpu, governor_buf);
+
+ sprintf(path, "/sys/devices/system/cpu/cpufreq/boost");
+ input = fopen(path, "r");
+ if (input != NULL) {
+ fscanf(input, "%d", &turbo);
+ fprintf(outf, "cpufreq boost: %d\n", turbo);
+ fclose(input);
+ }
+
+ sprintf(path, "/sys/devices/system/cpu/intel_pstate/no_turbo");
+ input = fopen(path, "r");
+ if (input != NULL) {
+ fscanf(input, "%d", &turbo);
+ fprintf(outf, "cpufreq intel_pstate no_turbo: %d\n", turbo);
+ fclose(input);
+ }
+}
+
/*
* print_epb()
case INTEL_FAM6_BROADWELL_CORE: /* BDW */
case INTEL_FAM6_BROADWELL_GT3E: /* BDW */
do_rapl = RAPL_PKG | RAPL_CORES | RAPL_CORE_POLICY | RAPL_GFX | RAPL_PKG_POWER_INFO;
+ if (rapl_joules) {
+ BIC_PRESENT(BIC_Pkg_J);
+ BIC_PRESENT(BIC_Cor_J);
+ BIC_PRESENT(BIC_GFX_J);
+ } else {
+ BIC_PRESENT(BIC_PkgWatt);
+ BIC_PRESENT(BIC_CorWatt);
+ BIC_PRESENT(BIC_GFXWatt);
+ }
break;
case INTEL_FAM6_ATOM_GOLDMONT: /* BXT */
+ case INTEL_FAM6_ATOM_GEMINI_LAKE:
do_rapl = RAPL_PKG | RAPL_PKG_POWER_INFO;
+ if (rapl_joules)
+ BIC_PRESENT(BIC_Pkg_J);
+ else
+ BIC_PRESENT(BIC_PkgWatt);
break;
case INTEL_FAM6_SKYLAKE_MOBILE: /* SKL */
case INTEL_FAM6_SKYLAKE_DESKTOP: /* SKL */
case INTEL_FAM6_KABYLAKE_MOBILE: /* KBL */
case INTEL_FAM6_KABYLAKE_DESKTOP: /* KBL */
do_rapl = RAPL_PKG | RAPL_DRAM | RAPL_DRAM_PERF_STATUS | RAPL_PKG_PERF_STATUS | RAPL_PKG_POWER_INFO;
+ BIC_PRESENT(BIC_PKG__);
+ BIC_PRESENT(BIC_RAM__);
+ if (rapl_joules) {
+ BIC_PRESENT(BIC_Pkg_J);
+ BIC_PRESENT(BIC_Cor_J);
+ BIC_PRESENT(BIC_RAM_J);
+ } else {
+ BIC_PRESENT(BIC_PkgWatt);
+ BIC_PRESENT(BIC_CorWatt);
+ BIC_PRESENT(BIC_RAMWatt);
+ }
break;
case INTEL_FAM6_HASWELL_X: /* HSX */
case INTEL_FAM6_BROADWELL_X: /* BDX */
case INTEL_FAM6_XEON_PHI_KNL: /* KNL */
case INTEL_FAM6_XEON_PHI_KNM:
do_rapl = RAPL_PKG | RAPL_DRAM | RAPL_DRAM_POWER_INFO | RAPL_DRAM_PERF_STATUS | RAPL_PKG_PERF_STATUS | RAPL_PKG_POWER_INFO;
+ BIC_PRESENT(BIC_PKG__);
+ BIC_PRESENT(BIC_RAM__);
+ if (rapl_joules) {
+ BIC_PRESENT(BIC_Pkg_J);
+ BIC_PRESENT(BIC_RAM_J);
+ } else {
+ BIC_PRESENT(BIC_PkgWatt);
+ BIC_PRESENT(BIC_RAMWatt);
+ }
break;
case INTEL_FAM6_SANDYBRIDGE_X:
case INTEL_FAM6_IVYBRIDGE_X:
do_rapl = RAPL_PKG | RAPL_CORES | RAPL_CORE_POLICY | RAPL_DRAM | RAPL_DRAM_POWER_INFO | RAPL_PKG_PERF_STATUS | RAPL_DRAM_PERF_STATUS | RAPL_PKG_POWER_INFO;
+ BIC_PRESENT(BIC_PKG__);
+ BIC_PRESENT(BIC_RAM__);
+ if (rapl_joules) {
+ BIC_PRESENT(BIC_Pkg_J);
+ BIC_PRESENT(BIC_Cor_J);
+ BIC_PRESENT(BIC_RAM_J);
+ } else {
+ BIC_PRESENT(BIC_PkgWatt);
+ BIC_PRESENT(BIC_CorWatt);
+ BIC_PRESENT(BIC_RAMWatt);
+ }
break;
case INTEL_FAM6_ATOM_SILVERMONT1: /* BYT */
case INTEL_FAM6_ATOM_SILVERMONT2: /* AVN */
do_rapl = RAPL_PKG | RAPL_CORES;
+ if (rapl_joules) {
+ BIC_PRESENT(BIC_Pkg_J);
+ BIC_PRESENT(BIC_Cor_J);
+ } else {
+ BIC_PRESENT(BIC_PkgWatt);
+ BIC_PRESENT(BIC_CorWatt);
+ }
break;
case INTEL_FAM6_ATOM_DENVERTON: /* DNV */
do_rapl = RAPL_PKG | RAPL_DRAM | RAPL_DRAM_POWER_INFO | RAPL_DRAM_PERF_STATUS | RAPL_PKG_PERF_STATUS | RAPL_PKG_POWER_INFO | RAPL_CORES_ENERGY_STATUS;
+ BIC_PRESENT(BIC_PKG__);
+ BIC_PRESENT(BIC_RAM__);
+ if (rapl_joules) {
+ BIC_PRESENT(BIC_Pkg_J);
+ BIC_PRESENT(BIC_Cor_J);
+ BIC_PRESENT(BIC_RAM_J);
+ } else {
+ BIC_PRESENT(BIC_PkgWatt);
+ BIC_PRESENT(BIC_CorWatt);
+ BIC_PRESENT(BIC_RAMWatt);
+ }
break;
default:
return;
tdp = get_tdp(model);
rapl_joule_counter_range = 0xFFFFFFFF * rapl_energy_units / tdp;
- if (debug)
+ if (!quiet)
fprintf(outf, "RAPL: %.0f sec. Joule Counter Range, at %.0f Watts\n", rapl_joule_counter_range, tdp);
return;
if (get_msr(cpu, MSR_RAPL_POWER_UNIT, &msr))
return -1;
- if (debug) {
- fprintf(outf, "cpu%d: MSR_RAPL_POWER_UNIT: 0x%08llx "
- "(%f Watts, %f Joules, %f sec.)\n", cpu, msr,
- rapl_power_units, rapl_energy_units, rapl_time_units);
- }
+ fprintf(outf, "cpu%d: MSR_RAPL_POWER_UNIT: 0x%08llx (%f Watts, %f Joules, %f sec.)\n", cpu, msr,
+ rapl_power_units, rapl_energy_units, rapl_time_units);
+
if (do_rapl & RAPL_PKG_POWER_INFO) {
if (get_msr(cpu, MSR_PKG_POWER_INFO, &msr))
return -9;
fprintf(outf, "cpu%d: MSR_PKG_POWER_LIMIT: 0x%08llx (%slocked)\n",
- cpu, msr, (msr >> 63) & 1 ? "": "UN");
+ cpu, msr, (msr >> 63) & 1 ? "" : "UN");
print_power_limit_msr(cpu, msr, "PKG Limit #1");
fprintf(outf, "cpu%d: PKG Limit #2: %sabled (%f Watts, %f* sec, clamp %sabled)\n",
if (get_msr(cpu, MSR_DRAM_POWER_LIMIT, &msr))
return -9;
fprintf(outf, "cpu%d: MSR_DRAM_POWER_LIMIT: 0x%08llx (%slocked)\n",
- cpu, msr, (msr >> 31) & 1 ? "": "UN");
+ cpu, msr, (msr >> 31) & 1 ? "" : "UN");
print_power_limit_msr(cpu, msr, "DRAM Limit");
}
if (do_rapl & RAPL_CORE_POLICY) {
- if (debug) {
- if (get_msr(cpu, MSR_PP0_POLICY, &msr))
- return -7;
+ if (get_msr(cpu, MSR_PP0_POLICY, &msr))
+ return -7;
- fprintf(outf, "cpu%d: MSR_PP0_POLICY: %lld\n", cpu, msr & 0xF);
- }
+ fprintf(outf, "cpu%d: MSR_PP0_POLICY: %lld\n", cpu, msr & 0xF);
}
if (do_rapl & RAPL_CORES_POWER_LIMIT) {
- if (debug) {
- if (get_msr(cpu, MSR_PP0_POWER_LIMIT, &msr))
- return -9;
- fprintf(outf, "cpu%d: MSR_PP0_POWER_LIMIT: 0x%08llx (%slocked)\n",
- cpu, msr, (msr >> 31) & 1 ? "": "UN");
- print_power_limit_msr(cpu, msr, "Cores Limit");
- }
+ if (get_msr(cpu, MSR_PP0_POWER_LIMIT, &msr))
+ return -9;
+ fprintf(outf, "cpu%d: MSR_PP0_POWER_LIMIT: 0x%08llx (%slocked)\n",
+ cpu, msr, (msr >> 31) & 1 ? "" : "UN");
+ print_power_limit_msr(cpu, msr, "Cores Limit");
}
if (do_rapl & RAPL_GFX) {
- if (debug) {
- if (get_msr(cpu, MSR_PP1_POLICY, &msr))
- return -8;
+ if (get_msr(cpu, MSR_PP1_POLICY, &msr))
+ return -8;
- fprintf(outf, "cpu%d: MSR_PP1_POLICY: %lld\n", cpu, msr & 0xF);
+ fprintf(outf, "cpu%d: MSR_PP1_POLICY: %lld\n", cpu, msr & 0xF);
- if (get_msr(cpu, MSR_PP1_POWER_LIMIT, &msr))
- return -9;
- fprintf(outf, "cpu%d: MSR_PP1_POWER_LIMIT: 0x%08llx (%slocked)\n",
- cpu, msr, (msr >> 31) & 1 ? "": "UN");
- print_power_limit_msr(cpu, msr, "GFX Limit");
- }
+ if (get_msr(cpu, MSR_PP1_POWER_LIMIT, &msr))
+ return -9;
+ fprintf(outf, "cpu%d: MSR_PP1_POWER_LIMIT: 0x%08llx (%slocked)\n",
+ cpu, msr, (msr >> 31) & 1 ? "" : "UN");
+ print_power_limit_msr(cpu, msr, "GFX Limit");
}
return 0;
}
case INTEL_FAM6_KABYLAKE_DESKTOP: /* KBL */
case INTEL_FAM6_SKYLAKE_X: /* SKX */
case INTEL_FAM6_ATOM_GOLDMONT: /* BXT */
+ case INTEL_FAM6_ATOM_GEMINI_LAKE:
case INTEL_FAM6_ATOM_DENVERTON: /* DNV */
return 1;
}
case INTEL_FAM6_KABYLAKE_MOBILE: /* KBL */
case INTEL_FAM6_KABYLAKE_DESKTOP: /* KBL */
case INTEL_FAM6_ATOM_GOLDMONT: /* BXT */
+ case INTEL_FAM6_ATOM_GEMINI_LAKE:
return 1;
}
return 0;
return 0;
}
-
-
int is_slm(unsigned int family, unsigned int model)
{
if (!genuine_intel)
}
freq = slm_freq_table[i];
- fprintf(outf, "SLM BCLK: %.1f Mhz\n", freq);
+ if (!quiet)
+ fprintf(outf, "SLM BCLK: %.1f Mhz\n", freq);
return freq;
}
target_c_local = (msr >> 16) & 0xFF;
- if (debug)
+ if (!quiet)
fprintf(outf, "cpu%d: MSR_IA32_TEMPERATURE_TARGET: 0x%08llx (%d C)\n",
cpu, msr, target_c_local);
unsigned long long msr;
if (!get_msr(base_cpu, MSR_IA32_MISC_ENABLE, &msr))
- fprintf(outf, "cpu%d: MSR_IA32_MISC_ENABLE: 0x%08llx (%s %s %s)\n",
+ fprintf(outf, "cpu%d: MSR_IA32_MISC_ENABLE: 0x%08llx (%sTCC %sEIST %sMWAIT %sPREFETCH %sTURBO)\n",
base_cpu, msr,
- msr & (1 << 3) ? "TCC" : "",
- msr & (1 << 16) ? "EIST" : "",
- msr & (1 << 18) ? "MONITOR" : "");
+ msr & MSR_IA32_MISC_ENABLE_TM1 ? "" : "No-",
+ msr & MSR_IA32_MISC_ENABLE_ENHANCED_SPEEDSTEP ? "" : "No-",
+ msr & MSR_IA32_MISC_ENABLE_MWAIT ? "No-" : "",
+ msr & MSR_IA32_MISC_ENABLE_PREFETCH_DISABLE ? "No-" : "",
+ msr & MSR_IA32_MISC_ENABLE_TURBO_DISABLE ? "No-" : "");
}
+void decode_misc_feature_control(void)
+{
+ unsigned long long msr;
+
+ if (!has_misc_feature_control)
+ return;
+
+ if (!get_msr(base_cpu, MSR_MISC_FEATURE_CONTROL, &msr))
+ fprintf(outf, "cpu%d: MSR_MISC_FEATURE_CONTROL: 0x%08llx (%sL2-Prefetch %sL2-Prefetch-pair %sL1-Prefetch %sL1-IP-Prefetch)\n",
+ base_cpu, msr,
+ msr & (0 << 0) ? "No-" : "",
+ msr & (1 << 0) ? "No-" : "",
+ msr & (2 << 0) ? "No-" : "",
+ msr & (3 << 0) ? "No-" : "");
+}
/*
* Decode MSR_MISC_PWR_MGMT
*
if (!do_nhm_platform_info)
return;
+ if (no_MSR_MISC_PWR_MGMT)
+ return;
+
if (!get_msr(base_cpu, MSR_MISC_PWR_MGMT, &msr))
fprintf(outf, "cpu%d: MSR_MISC_PWR_MGMT: 0x%08llx (%sable-EIST_Coordination %sable-EPB %sable-OOB)\n",
base_cpu, msr,
msr & (1 << 1) ? "EN" : "DIS",
msr & (1 << 8) ? "EN" : "DIS");
}
+/*
+ * Decode MSR_CC6_DEMOTION_POLICY_CONFIG, MSR_MC6_DEMOTION_POLICY_CONFIG
+ *
+ * This MSRs are present on Silvermont processors,
+ * Intel Atom processor E3000 series (Baytrail), and friends.
+ */
+void decode_c6_demotion_policy_msr(void)
+{
+ unsigned long long msr;
+
+ if (!get_msr(base_cpu, MSR_CC6_DEMOTION_POLICY_CONFIG, &msr))
+ fprintf(outf, "cpu%d: MSR_CC6_DEMOTION_POLICY_CONFIG: 0x%08llx (%sable-CC6-Demotion)\n",
+ base_cpu, msr, msr & (1 << 0) ? "EN" : "DIS");
+
+ if (!get_msr(base_cpu, MSR_MC6_DEMOTION_POLICY_CONFIG, &msr))
+ fprintf(outf, "cpu%d: MSR_MC6_DEMOTION_POLICY_CONFIG: 0x%08llx (%sable-MC6-Demotion)\n",
+ base_cpu, msr, msr & (1 << 0) ? "EN" : "DIS");
+}
void process_cpuid()
{
unsigned int eax, ebx, ecx, edx, max_level, max_extended_level;
unsigned int fms, family, model, stepping;
+ unsigned int has_turbo;
eax = ebx = ecx = edx = 0;
if (ebx == 0x756e6547 && edx == 0x49656e69 && ecx == 0x6c65746e)
genuine_intel = 1;
- if (debug)
+ if (!quiet)
fprintf(outf, "CPUID(0): %.4s%.4s%.4s ",
(char *)&ebx, (char *)&edx, (char *)&ecx);
if (family == 6 || family == 0xf)
model += ((fms >> 16) & 0xf) << 4;
- if (debug) {
+ if (!quiet) {
fprintf(outf, "%d CPUID levels; family:model:stepping 0x%x:%x:%x (%d:%d:%d)\n",
max_level, family, model, stepping, family, model, stepping);
fprintf(outf, "CPUID(1): %s %s %s %s %s %s %s %s %s\n",
__cpuid(0x6, eax, ebx, ecx, edx);
has_aperf = ecx & (1 << 0);
+ if (has_aperf) {
+ BIC_PRESENT(BIC_Avg_MHz);
+ BIC_PRESENT(BIC_Busy);
+ BIC_PRESENT(BIC_Bzy_MHz);
+ }
do_dts = eax & (1 << 0);
+ if (do_dts)
+ BIC_PRESENT(BIC_CoreTmp);
+ has_turbo = eax & (1 << 1);
do_ptm = eax & (1 << 6);
+ if (do_ptm)
+ BIC_PRESENT(BIC_PkgTmp);
has_hwp = eax & (1 << 7);
has_hwp_notify = eax & (1 << 8);
has_hwp_activity_window = eax & (1 << 9);
has_hwp_pkg = eax & (1 << 11);
has_epb = ecx & (1 << 3);
- if (debug)
- fprintf(outf, "CPUID(6): %sAPERF, %sDTS, %sPTM, %sHWP, "
+ if (!quiet)
+ fprintf(outf, "CPUID(6): %sAPERF, %sTURBO, %sDTS, %sPTM, %sHWP, "
"%sHWPnotify, %sHWPwindow, %sHWPepp, %sHWPpkg, %sEPB\n",
has_aperf ? "" : "No-",
+ has_turbo ? "" : "No-",
do_dts ? "" : "No-",
do_ptm ? "" : "No-",
has_hwp ? "" : "No-",
has_hwp_pkg ? "" : "No-",
has_epb ? "" : "No-");
- if (debug)
+ if (!quiet)
decode_misc_enable_msr();
- if (max_level >= 0x7 && debug) {
+
+ if (max_level >= 0x7 && !quiet) {
int has_sgx;
ecx = 0;
if (ebx_tsc != 0) {
- if (debug && (ebx != 0))
+ if (!quiet && (ebx != 0))
fprintf(outf, "CPUID(0x15): eax_crystal: %d ebx_tsc: %d ecx_crystal_hz: %d\n",
eax_crystal, ebx_tsc, crystal_hz);
crystal_hz = 25000000; /* 25.0 MHz */
break;
case INTEL_FAM6_ATOM_GOLDMONT: /* BXT */
+ case INTEL_FAM6_ATOM_GEMINI_LAKE:
crystal_hz = 19200000; /* 19.2 MHz */
break;
default:
if (crystal_hz) {
tsc_hz = (unsigned long long) crystal_hz * ebx_tsc / eax_crystal;
- if (debug)
+ if (!quiet)
fprintf(outf, "TSC: %lld MHz (%d Hz * %d / %d / 1000000)\n",
tsc_hz / 1000000, crystal_hz, ebx_tsc, eax_crystal);
}
base_mhz = max_mhz = bus_mhz = edx = 0;
__cpuid(0x16, base_mhz, max_mhz, bus_mhz, edx);
- if (debug)
+ if (!quiet)
fprintf(outf, "CPUID(0x16): base_mhz: %d max_mhz: %d bus_mhz: %d\n",
base_mhz, max_mhz, bus_mhz);
}
if (has_aperf)
aperf_mperf_multiplier = get_aperf_mperf_multiplier(family, model);
- do_nhm_platform_info = do_nhm_cstates = do_smi = probe_nhm_msrs(family, model);
+ BIC_PRESENT(BIC_IRQ);
+ BIC_PRESENT(BIC_TSC_MHz);
+
+ if (probe_nhm_msrs(family, model)) {
+ do_nhm_platform_info = 1;
+ BIC_PRESENT(BIC_CPU_c1);
+ BIC_PRESENT(BIC_CPU_c3);
+ BIC_PRESENT(BIC_CPU_c6);
+ BIC_PRESENT(BIC_SMI);
+ }
do_snb_cstates = has_snb_msrs(family, model);
+
+ if (do_snb_cstates)
+ BIC_PRESENT(BIC_CPU_c7);
+
do_irtl_snb = has_snb_msrs(family, model);
- do_pc2 = do_snb_cstates && (pkg_cstate_limit >= PCL__2);
- do_pc3 = (pkg_cstate_limit >= PCL__3);
- do_pc6 = (pkg_cstate_limit >= PCL__6);
- do_pc7 = do_snb_cstates && (pkg_cstate_limit >= PCL__7);
- do_c8_c9_c10 = has_hsw_msrs(family, model);
+ if (do_snb_cstates && (pkg_cstate_limit >= PCL__2))
+ BIC_PRESENT(BIC_Pkgpc2);
+ if (pkg_cstate_limit >= PCL__3)
+ BIC_PRESENT(BIC_Pkgpc3);
+ if (pkg_cstate_limit >= PCL__6)
+ BIC_PRESENT(BIC_Pkgpc6);
+ if (do_snb_cstates && (pkg_cstate_limit >= PCL__7))
+ BIC_PRESENT(BIC_Pkgpc7);
+ if (has_slv_msrs(family, model)) {
+ BIC_NOT_PRESENT(BIC_Pkgpc2);
+ BIC_NOT_PRESENT(BIC_Pkgpc3);
+ BIC_PRESENT(BIC_Pkgpc6);
+ BIC_NOT_PRESENT(BIC_Pkgpc7);
+ BIC_PRESENT(BIC_Mod_c6);
+ use_c1_residency_msr = 1;
+ }
+ if (is_dnv(family, model)) {
+ BIC_PRESENT(BIC_CPU_c1);
+ BIC_NOT_PRESENT(BIC_CPU_c3);
+ BIC_NOT_PRESENT(BIC_Pkgpc3);
+ BIC_NOT_PRESENT(BIC_CPU_c7);
+ BIC_NOT_PRESENT(BIC_Pkgpc7);
+ use_c1_residency_msr = 1;
+ }
+ if (is_skx(family, model)) {
+ BIC_NOT_PRESENT(BIC_CPU_c3);
+ BIC_NOT_PRESENT(BIC_Pkgpc3);
+ BIC_NOT_PRESENT(BIC_CPU_c7);
+ BIC_NOT_PRESENT(BIC_Pkgpc7);
+ }
+ if (is_bdx(family, model)) {
+ BIC_NOT_PRESENT(BIC_CPU_c7);
+ BIC_NOT_PRESENT(BIC_Pkgpc7);
+ }
+ if (has_hsw_msrs(family, model)) {
+ BIC_PRESENT(BIC_Pkgpc8);
+ BIC_PRESENT(BIC_Pkgpc9);
+ BIC_PRESENT(BIC_Pkgpc10);
+ }
do_irtl_hsw = has_hsw_msrs(family, model);
do_skl_residency = has_skl_msrs(family, model);
do_slm_cstates = is_slm(family, model);
do_knl_cstates = is_knl(family, model);
- if (debug)
+ if (!quiet)
decode_misc_pwr_mgmt_msr();
+ if (!quiet && has_slv_msrs(family, model))
+ decode_c6_demotion_policy_msr();
+
rapl_probe(family, model);
perf_limit_reasons_probe(family, model);
- if (debug)
+ if (!quiet)
dump_cstate_pstate_config_info(family, model);
+ if (!quiet)
+ dump_sysfs_cstate_config();
+ if (!quiet)
+ dump_sysfs_pstate_config();
+
if (has_skl_msrs(family, model))
calculate_tsc_tweak();
- do_gfx_rc6_ms = !access("/sys/class/drm/card0/power/rc6_residency_ms", R_OK);
+ if (!access("/sys/class/drm/card0/power/rc6_residency_ms", R_OK))
+ BIC_PRESENT(BIC_GFX_rc6);
- do_gfx_mhz = !access("/sys/class/graphics/fb0/device/drm/card0/gt_cur_freq_mhz", R_OK);
+ if (!access("/sys/class/graphics/fb0/device/drm/card0/gt_cur_freq_mhz", R_OK))
+ BIC_PRESENT(BIC_GFXMHz);
- return;
-}
+ if (!quiet)
+ decode_misc_feature_control();
-void help()
-{
- fprintf(outf,
- "Usage: turbostat [OPTIONS][(--interval seconds) | COMMAND ...]\n"
- "\n"
- "Turbostat forks the specified COMMAND and prints statistics\n"
- "when COMMAND completes.\n"
- "If no COMMAND is specified, turbostat wakes every 5-seconds\n"
- "to print statistics, until interrupted.\n"
- "--add add a counter\n"
- " eg. --add msr0x10,u64,cpu,delta,MY_TSC\n"
- "--debug run in \"debug\" mode\n"
- "--interval sec Override default 5-second measurement interval\n"
- "--help print this help message\n"
- "--out file create or truncate \"file\" for all output\n"
- "--version print version information\n"
- "\n"
- "For more help, run \"man turbostat\"\n");
+ return;
}
topo.max_cpu_num = 0;
for_all_proc_cpus(count_cpus);
if (!summary_only && topo.num_cpus > 1)
- show_cpu = 1;
+ BIC_PRESENT(BIC_CPU);
if (debug > 1)
fprintf(outf, "num_cpus %d max_cpu_num %d\n", topo.num_cpus, topo.max_cpu_num);
CPU_ZERO_S(cpu_present_setsize, cpu_present_set);
for_all_proc_cpus(mark_cpu_present);
+ /*
+ * Validate that all cpus in cpu_subset are also in cpu_present_set
+ */
+ for (i = 0; i < CPU_SUBSET_MAXCPUS; ++i) {
+ if (CPU_ISSET_S(i, cpu_subset_size, cpu_subset))
+ if (!CPU_ISSET_S(i, cpu_present_setsize, cpu_present_set))
+ err(1, "cpu%d not present", i);
+ }
+
/*
* Allocate and initialize cpu_affinity_set
*/
if (debug > 1)
fprintf(outf, "max_core_id %d, sizing for %d cores per package\n",
max_core_id, topo.num_cores_per_pkg);
- if (debug && !summary_only && topo.num_cores_per_pkg > 1)
- show_core = 1;
+ if (!summary_only && topo.num_cores_per_pkg > 1)
+ BIC_PRESENT(BIC_Core);
topo.num_packages = max_package_id + 1;
if (debug > 1)
fprintf(outf, "max_package_id %d, sizing for %d packages\n",
max_package_id, topo.num_packages);
- if (debug && !summary_only && topo.num_packages > 1)
- show_pkg = 1;
+ if (!summary_only && topo.num_packages > 1)
+ BIC_PRESENT(BIC_Package);
topo.num_threads_per_core = max_siblings;
if (debug > 1)
int i;
*t = calloc(topo.num_threads_per_core * topo.num_cores_per_pkg *
- topo.num_packages, sizeof(struct thread_data) + sys.thread_counter_bytes);
+ topo.num_packages, sizeof(struct thread_data));
if (*t == NULL)
goto error;
(*t)[i].cpu_id = -1;
*c = calloc(topo.num_cores_per_pkg * topo.num_packages,
- sizeof(struct core_data) + sys.core_counter_bytes);
+ sizeof(struct core_data));
if (*c == NULL)
goto error;
for (i = 0; i < topo.num_cores_per_pkg * topo.num_packages; i++)
(*c)[i].core_id = -1;
- *p = calloc(topo.num_packages, sizeof(struct pkg_data) + sys.package_counter_bytes);
+ *p = calloc(topo.num_packages, sizeof(struct pkg_data));
if (*p == NULL)
goto error;
process_cpuid();
- if (debug)
+ if (!quiet)
for_all_cpus(print_hwp, ODD_COUNTERS);
- if (debug)
+ if (!quiet)
for_all_cpus(print_epb, ODD_COUNTERS);
- if (debug)
+ if (!quiet)
for_all_cpus(print_perf_limit, ODD_COUNTERS);
- if (debug)
+ if (!quiet)
for_all_cpus(print_rapl, ODD_COUNTERS);
for_all_cpus(set_temperature_target, ODD_COUNTERS);
- if (debug)
+ if (!quiet)
for_all_cpus(print_thermal, ODD_COUNTERS);
- if (debug && do_irtl_snb)
+ if (!quiet && do_irtl_snb)
print_irtl();
}
pid_t child_pid;
int status;
+ snapshot_proc_sysfs_files();
status = for_all_cpus(get_counters, EVEN_COUNTERS);
if (status)
exit(status);
if (!child_pid) {
/* child */
execvp(argv[0], argv);
+ err(errno, "exec %s", argv[0]);
} else {
/* parent */
* n.b. fork_it() does not check for errors from for_all_cpus()
* because re-starting is problematic when forking
*/
+ snapshot_proc_sysfs_files();
for_all_cpus(get_counters, ODD_COUNTERS);
gettimeofday(&tv_odd, (struct timezone *)NULL);
timersub(&tv_odd, &tv_even, &tv_delta);
{
int status;
+ snapshot_proc_sysfs_files();
status = for_all_cpus(get_counters, ODD_COUNTERS);
if (status)
return status;
}
void print_version() {
- fprintf(outf, "turbostat version 4.16 24 Dec 2016"
+ fprintf(outf, "turbostat version 17.02.24"
" - Len Brown <lenb@kernel.org>\n");
}
-int add_counter(unsigned int msr_num, char *name, unsigned int width,
- enum counter_scope scope, enum counter_type type,
- enum counter_format format)
+int add_counter(unsigned int msr_num, char *path, char *name,
+ unsigned int width, enum counter_scope scope,
+ enum counter_type type, enum counter_format format, int flags)
{
struct msr_counter *msrp;
msrp->msr_num = msr_num;
strncpy(msrp->name, name, NAME_BYTES);
+ if (path)
+ strncpy(msrp->path, path, PATH_BYTES);
msrp->width = width;
msrp->type = type;
msrp->format = format;
+ msrp->flags = flags;
switch (scope) {
case SCOPE_CPU:
- sys.thread_counter_bytes += 64;
msrp->next = sys.tp;
sys.tp = msrp;
- sys.thread_counter_bytes += sizeof(unsigned long long);
+ sys.added_thread_counters++;
+ if (sys.added_thread_counters > MAX_ADDED_COUNTERS) {
+ fprintf(stderr, "exceeded max %d added thread counters\n",
+ MAX_ADDED_COUNTERS);
+ exit(-1);
+ }
break;
case SCOPE_CORE:
- sys.core_counter_bytes += 64;
msrp->next = sys.cp;
sys.cp = msrp;
- sys.core_counter_bytes += sizeof(unsigned long long);
+ sys.added_core_counters++;
+ if (sys.added_core_counters > MAX_ADDED_COUNTERS) {
+ fprintf(stderr, "exceeded max %d added core counters\n",
+ MAX_ADDED_COUNTERS);
+ exit(-1);
+ }
break;
case SCOPE_PACKAGE:
- sys.package_counter_bytes += 64;
msrp->next = sys.pp;
sys.pp = msrp;
- sys.package_counter_bytes += sizeof(unsigned long long);
+ sys.added_package_counters++;
+ if (sys.added_package_counters > MAX_ADDED_COUNTERS) {
+ fprintf(stderr, "exceeded max %d added package counters\n",
+ MAX_ADDED_COUNTERS);
+ exit(-1);
+ }
break;
}
void parse_add_command(char *add_command)
{
int msr_num = 0;
- char name_buffer[NAME_BYTES];
+ char *path = NULL;
+ char name_buffer[NAME_BYTES] = "";
int width = 64;
int fail = 0;
enum counter_scope scope = SCOPE_CPU;
if (sscanf(add_command, "msr%d", &msr_num) == 1)
goto next;
+ if (*add_command == '/') {
+ path = add_command;
+ goto next;
+ }
+
if (sscanf(add_command, "u%d", &width) == 1) {
if ((width == 32) || (width == 64))
goto next;
type = COUNTER_SECONDS;
goto next;
}
+ if (!strncmp(add_command, "usec", strlen("usec"))) {
+ type = COUNTER_USEC;
+ goto next;
+ }
if (!strncmp(add_command, "raw", strlen("raw"))) {
format = FORMAT_RAW;
goto next;
next:
add_command = strchr(add_command, ',');
- if (add_command)
+ if (add_command) {
+ *add_command = '\0';
add_command++;
+ }
}
- if (msr_num == 0) {
- fprintf(stderr, "--add: (msrDDD | msr0xXXX) required\n");
+ if ((msr_num == 0) && (path == NULL)) {
+ fprintf(stderr, "--add: (msrDDD | msr0xXXX | /path_to_counter ) required\n");
fail++;
}
/* generate default column header */
if (*name_buffer == '\0') {
- if (format == FORMAT_RAW) {
- if (width == 32)
- sprintf(name_buffer, "msr%d", msr_num);
- else
- sprintf(name_buffer, "MSR%d", msr_num);
- } else if (format == FORMAT_DELTA) {
- if (width == 32)
- sprintf(name_buffer, "cnt%d", msr_num);
- else
- sprintf(name_buffer, "CNT%d", msr_num);
- } else if (format == FORMAT_PERCENT) {
- if (width == 32)
- sprintf(name_buffer, "msr%d%%", msr_num);
- else
- sprintf(name_buffer, "MSR%d%%", msr_num);
- }
+ if (width == 32)
+ sprintf(name_buffer, "M0x%x%s", msr_num, format == FORMAT_PERCENT ? "%" : "");
+ else
+ sprintf(name_buffer, "M0X%x%s", msr_num, format == FORMAT_PERCENT ? "%" : "");
}
- if (add_counter(msr_num, name_buffer, width, scope, type, format))
+ if (add_counter(msr_num, path, name_buffer, width, scope, type, format, 0))
fail++;
if (fail) {
exit(1);
}
}
+
+int is_deferred_skip(char *name)
+{
+ int i;
+
+ for (i = 0; i < deferred_skip_index; ++i)
+ if (!strcmp(name, deferred_skip_names[i]))
+ return 1;
+ return 0;
+}
+
+void probe_sysfs(void)
+{
+ char path[64];
+ char name_buf[16];
+ FILE *input;
+ int state;
+ char *sp;
+
+ if (!DO_BIC(BIC_sysfs))
+ return;
+
+ for (state = 10; state > 0; --state) {
+
+ sprintf(path, "/sys/devices/system/cpu/cpu%d/cpuidle/state%d/name",
+ base_cpu, state);
+ input = fopen(path, "r");
+ if (input == NULL)
+ continue;
+ fgets(name_buf, sizeof(name_buf), input);
+
+ /* truncate "C1-HSW\n" to "C1", or truncate "C1\n" to "C1" */
+ sp = strchr(name_buf, '-');
+ if (!sp)
+ sp = strchrnul(name_buf, '\n');
+ *sp = '%';
+ *(sp + 1) = '\0';
+
+ fclose(input);
+
+ sprintf(path, "cpuidle/state%d/time", state);
+
+ if (is_deferred_skip(name_buf))
+ continue;
+
+ add_counter(0, path, name_buf, 64, SCOPE_CPU, COUNTER_USEC,
+ FORMAT_PERCENT, SYSFS_PERCPU);
+ }
+
+ for (state = 10; state > 0; --state) {
+
+ sprintf(path, "/sys/devices/system/cpu/cpu%d/cpuidle/state%d/name",
+ base_cpu, state);
+ input = fopen(path, "r");
+ if (input == NULL)
+ continue;
+ fgets(name_buf, sizeof(name_buf), input);
+ /* truncate "C1-HSW\n" to "C1", or truncate "C1\n" to "C1" */
+ sp = strchr(name_buf, '-');
+ if (!sp)
+ sp = strchrnul(name_buf, '\n');
+ *sp = '\0';
+ fclose(input);
+
+ sprintf(path, "cpuidle/state%d/usage", state);
+
+ if (is_deferred_skip(name_buf))
+ continue;
+
+ add_counter(0, path, name_buf, 64, SCOPE_CPU, COUNTER_ITEMS,
+ FORMAT_DELTA, SYSFS_PERCPU);
+ }
+
+}
+
+
+/*
+ * parse cpuset with following syntax
+ * 1,2,4..6,8-10 and set bits in cpu_subset
+ */
+void parse_cpu_command(char *optarg)
+{
+ unsigned int start, end;
+ char *next;
+
+ if (!strcmp(optarg, "core")) {
+ if (cpu_subset)
+ goto error;
+ show_core_only++;
+ return;
+ }
+ if (!strcmp(optarg, "package")) {
+ if (cpu_subset)
+ goto error;
+ show_pkg_only++;
+ return;
+ }
+ if (show_core_only || show_pkg_only)
+ goto error;
+
+ cpu_subset = CPU_ALLOC(CPU_SUBSET_MAXCPUS);
+ if (cpu_subset == NULL)
+ err(3, "CPU_ALLOC");
+ cpu_subset_size = CPU_ALLOC_SIZE(CPU_SUBSET_MAXCPUS);
+
+ CPU_ZERO_S(cpu_subset_size, cpu_subset);
+
+ next = optarg;
+
+ while (next && *next) {
+
+ if (*next == '-') /* no negative cpu numbers */
+ goto error;
+
+ start = strtoul(next, &next, 10);
+
+ if (start >= CPU_SUBSET_MAXCPUS)
+ goto error;
+ CPU_SET_S(start, cpu_subset_size, cpu_subset);
+
+ if (*next == '\0')
+ break;
+
+ if (*next == ',') {
+ next += 1;
+ continue;
+ }
+
+ if (*next == '-') {
+ next += 1; /* start range */
+ } else if (*next == '.') {
+ next += 1;
+ if (*next == '.')
+ next += 1; /* start range */
+ else
+ goto error;
+ }
+
+ end = strtoul(next, &next, 10);
+ if (end <= start)
+ goto error;
+
+ while (++start <= end) {
+ if (start >= CPU_SUBSET_MAXCPUS)
+ goto error;
+ CPU_SET_S(start, cpu_subset_size, cpu_subset);
+ }
+
+ if (*next == ',')
+ next += 1;
+ else if (*next != '\0')
+ goto error;
+ }
+
+ return;
+
+error:
+ fprintf(stderr, "\"--cpu %s\" malformed\n", optarg);
+ help();
+ exit(-1);
+}
+
+int shown;
+/*
+ * parse_show_hide() - process cmdline to set default counter action
+ */
+void parse_show_hide(char *optarg, enum show_hide_mode new_mode)
+{
+ /*
+ * --show: show only those specified
+ * The 1st invocation will clear and replace the enabled mask
+ * subsequent invocations can add to it.
+ */
+ if (new_mode == SHOW_LIST) {
+ if (shown == 0)
+ bic_enabled = bic_lookup(optarg, new_mode);
+ else
+ bic_enabled |= bic_lookup(optarg, new_mode);
+ shown = 1;
+
+ return;
+ }
+
+ /*
+ * --hide: do not show those specified
+ * multiple invocations simply clear more bits in enabled mask
+ */
+ bic_enabled &= ~bic_lookup(optarg, new_mode);
+
+}
+
void cmdline(int argc, char **argv)
{
int opt;
int option_index = 0;
static struct option long_options[] = {
{"add", required_argument, 0, 'a'},
+ {"cpu", required_argument, 0, 'c'},
{"Dump", no_argument, 0, 'D'},
- {"debug", no_argument, 0, 'd'},
+ {"debug", no_argument, 0, 'd'}, /* internal, not documented */
{"interval", required_argument, 0, 'i'},
{"help", no_argument, 0, 'h'},
+ {"hide", required_argument, 0, 'H'}, // meh, -h taken by --help
{"Joules", no_argument, 0, 'J'},
+ {"list", no_argument, 0, 'l'},
{"out", required_argument, 0, 'o'},
- {"Package", no_argument, 0, 'p'},
- {"processor", no_argument, 0, 'p'},
+ {"quiet", no_argument, 0, 'q'},
+ {"show", required_argument, 0, 's'},
{"Summary", no_argument, 0, 'S'},
{"TCC", required_argument, 0, 'T'},
{"version", no_argument, 0, 'v' },
progname = argv[0];
- while ((opt = getopt_long_only(argc, argv, "+C:c:Ddhi:JM:m:o:PpST:v",
+ while ((opt = getopt_long_only(argc, argv, "+C:c:Ddhi:JM:m:o:qST:v",
long_options, &option_index)) != -1) {
switch (opt) {
case 'a':
parse_add_command(optarg);
break;
+ case 'c':
+ parse_cpu_command(optarg);
+ break;
case 'D':
dump_only++;
break;
case 'd':
debug++;
break;
+ case 'H':
+ parse_show_hide(optarg, HIDE_LIST);
+ break;
case 'h':
default:
help();
case 'J':
rapl_joules++;
break;
+ case 'l':
+ list_header_only++;
+ quiet++;
+ break;
case 'o':
outf = fopen_or_die(optarg, "w");
break;
- case 'P':
- show_pkg_only++;
+ case 'q':
+ quiet = 1;
break;
- case 'p':
- show_core_only++;
+ case 's':
+ parse_show_hide(optarg, SHOW_LIST);
break;
case 'S':
summary_only++;
cmdline(argc, argv);
- if (debug)
+ if (!quiet)
print_version();
+ probe_sysfs();
+
turbostat_init();
/* dump counters and exit */
if (dump_only)
return get_and_dump_counters();
+ /* list header and exit */
+ if (list_header_only) {
+ print_header(",");
+ flush_output_stdout();
+ return 0;
+ }
+
/*
* if any params left, it must be a command to fork
*/
sub wait_for_input
{
my ($fp, $time) = @_;
+ my $start_time;
my $rin;
my $rout;
my $nr;
vec($rin, fileno($fp), 1) = 1;
vec($rin, fileno(\*STDIN), 1) = 1;
+ $start_time = time;
+
while (1) {
$nr = select($rout=$rin, undef, undef, $time);
- if ($nr <= 0) {
- return undef;
- }
+ last if ($nr <= 0);
# copy data from stdin to the console
if (vec($rout, fileno(\*STDIN), 1) == 1) {
- sysread(\*STDIN, $buf, 1000);
- syswrite($fp, $buf, 1000);
+ $nr = sysread(\*STDIN, $buf, 1000);
+ syswrite($fp, $buf, $nr) if ($nr > 0);
+ }
+
+ # The timeout is based on time waiting for the fp data
+ if (vec($rout, fileno($fp), 1) != 1) {
+ last if (defined($time) && (time - $start_time > $time));
next;
}
last if ($ch eq "\n");
}
- if (!length($line)) {
- return undef;
- }
+ last if (!length($line));
return $line;
}
+ return undef;
}
sub reboot_to {
memdev->range_index = 0+1;
memdev->region_index = 4+1;
memdev->region_size = SPA0_SIZE/2;
- memdev->region_offset = t->spa_set_dma[0];
+ memdev->region_offset = 1;
memdev->address = 0;
memdev->interleave_index = 0;
memdev->interleave_ways = 2;
memdev->range_index = 0+1;
memdev->region_index = 5+1;
memdev->region_size = SPA0_SIZE/2;
- memdev->region_offset = t->spa_set_dma[0] + SPA0_SIZE/2;
+ memdev->region_offset = (1 << 8);
memdev->address = 0;
memdev->interleave_index = 0;
memdev->interleave_ways = 2;
memdev->range_index = 1+1;
memdev->region_index = 4+1;
memdev->region_size = SPA1_SIZE/4;
- memdev->region_offset = t->spa_set_dma[1];
+ memdev->region_offset = (1 << 16);
memdev->address = SPA0_SIZE/2;
memdev->interleave_index = 0;
memdev->interleave_ways = 4;
memdev->range_index = 1+1;
memdev->region_index = 5+1;
memdev->region_size = SPA1_SIZE/4;
- memdev->region_offset = t->spa_set_dma[1] + SPA1_SIZE/4;
+ memdev->region_offset = (1 << 24);
memdev->address = SPA0_SIZE/2;
memdev->interleave_index = 0;
memdev->interleave_ways = 4;
memdev->range_index = 1+1;
memdev->region_index = 6+1;
memdev->region_size = SPA1_SIZE/4;
- memdev->region_offset = t->spa_set_dma[1] + 2*SPA1_SIZE/4;
+ memdev->region_offset = (1ULL << 32);
memdev->address = SPA0_SIZE/2;
memdev->interleave_index = 0;
memdev->interleave_ways = 4;
memdev->range_index = 1+1;
memdev->region_index = 7+1;
memdev->region_size = SPA1_SIZE/4;
- memdev->region_offset = t->spa_set_dma[1] + 3*SPA1_SIZE/4;
+ memdev->region_offset = (1ULL << 40);
memdev->address = SPA0_SIZE/2;
memdev->interleave_index = 0;
memdev->interleave_ways = 4;
memdev->range_index = 11+1;
memdev->region_index = 9+1;
memdev->region_size = SPA0_SIZE;
- memdev->region_offset = t->spa_set_dma[2];
+ memdev->region_offset = (1ULL << 48);
memdev->address = 0;
memdev->interleave_index = 0;
memdev->interleave_ways = 1;
CFLAGS += -I. -I../../include -g -O2 -Wall -D_LGPL_SOURCE -fsanitize=address
-LDFLAGS += -lpthread -lurcu
+LDFLAGS += -fsanitize=address
+LDLIBS+= -lpthread -lurcu
TARGETS = main idr-test multiorder
CORE_OFILES := radix-tree.o idr.o linux.o test.o find_bit.o
OFILES = main.o $(CORE_OFILES) regression1.o regression2.o regression3.o \
SHIFT=3
endif
+ifeq ($(BUILD), 32)
+ CFLAGS += -m32
+ LDFLAGS += -m32
+endif
+
targets: mapshift $(TARGETS)
main: $(OFILES)
- $(CC) $(CFLAGS) $(LDFLAGS) $^ -o main
idr-test: idr-test.o $(CORE_OFILES)
- $(CC) $(CFLAGS) $(LDFLAGS) $^ -o idr-test
multiorder: multiorder.o $(CORE_OFILES)
- $(CC) $(CFLAGS) $(LDFLAGS) $^ -o multiorder
clean:
$(RM) $(TARGETS) *.o radix-tree.c idr.c generated/map-shift.h
vpath %.c ../../lib
-$(OFILES): *.h */*.h generated/map-shift.h \
+$(OFILES): Makefile *.h */*.h generated/map-shift.h \
../../include/linux/*.h \
../../include/asm/*.h \
../../../include/linux/radix-tree.h \
.PHONY: mapshift
mapshift:
- @if ! grep -qw $(SHIFT) generated/map-shift.h; then \
+ @if ! grep -qws $(SHIFT) generated/map-shift.h; then \
echo "#define RADIX_TREE_MAP_SHIFT $(SHIFT)" > \
generated/map-shift.h; \
fi
#include <time.h>
#include "test.h"
+#define for_each_index(i, base, order) \
+ for (i = base; i < base + (1 << order); i++)
+
#define NSEC_PER_SEC 1000000000L
static long long benchmark_iter(struct radix_tree_root *root, bool tagged)
return nsec;
}
+static void benchmark_insert(struct radix_tree_root *root,
+ unsigned long size, unsigned long step, int order)
+{
+ struct timespec start, finish;
+ unsigned long index;
+ long long nsec;
+
+ clock_gettime(CLOCK_MONOTONIC, &start);
+
+ for (index = 0 ; index < size ; index += step)
+ item_insert_order(root, index, order);
+
+ clock_gettime(CLOCK_MONOTONIC, &finish);
+
+ nsec = (finish.tv_sec - start.tv_sec) * NSEC_PER_SEC +
+ (finish.tv_nsec - start.tv_nsec);
+
+ printv(2, "Size: %8ld, step: %8ld, order: %d, insertion: %15lld ns\n",
+ size, step, order, nsec);
+}
+
+static void benchmark_tagging(struct radix_tree_root *root,
+ unsigned long size, unsigned long step, int order)
+{
+ struct timespec start, finish;
+ unsigned long index;
+ long long nsec;
+
+ clock_gettime(CLOCK_MONOTONIC, &start);
+
+ for (index = 0 ; index < size ; index += step)
+ radix_tree_tag_set(root, index, 0);
+
+ clock_gettime(CLOCK_MONOTONIC, &finish);
+
+ nsec = (finish.tv_sec - start.tv_sec) * NSEC_PER_SEC +
+ (finish.tv_nsec - start.tv_nsec);
+
+ printv(2, "Size: %8ld, step: %8ld, order: %d, tagging: %17lld ns\n",
+ size, step, order, nsec);
+}
+
+static void benchmark_delete(struct radix_tree_root *root,
+ unsigned long size, unsigned long step, int order)
+{
+ struct timespec start, finish;
+ unsigned long index, i;
+ long long nsec;
+
+ clock_gettime(CLOCK_MONOTONIC, &start);
+
+ for (index = 0 ; index < size ; index += step)
+ for_each_index(i, index, order)
+ item_delete(root, i);
+
+ clock_gettime(CLOCK_MONOTONIC, &finish);
+
+ nsec = (finish.tv_sec - start.tv_sec) * NSEC_PER_SEC +
+ (finish.tv_nsec - start.tv_nsec);
+
+ printv(2, "Size: %8ld, step: %8ld, order: %d, deletion: %16lld ns\n",
+ size, step, order, nsec);
+}
+
static void benchmark_size(unsigned long size, unsigned long step, int order)
{
RADIX_TREE(tree, GFP_KERNEL);
long long normal, tagged;
- unsigned long index;
- for (index = 0 ; index < size ; index += step) {
- item_insert_order(&tree, index, order);
- radix_tree_tag_set(&tree, index, 0);
- }
+ benchmark_insert(&tree, size, step, order);
+ benchmark_tagging(&tree, size, step, order);
tagged = benchmark_iter(&tree, true);
normal = benchmark_iter(&tree, false);
- printv(2, "Size %ld, step %6ld, order %d tagged %10lld ns, normal %10lld ns\n",
- size, step, order, tagged, normal);
+ printv(2, "Size: %8ld, step: %8ld, order: %d, tagged iteration: %8lld ns\n",
+ size, step, order, tagged);
+ printv(2, "Size: %8ld, step: %8ld, order: %d, normal iteration: %8lld ns\n",
+ size, step, order, normal);
+
+ benchmark_delete(&tree, size, step, order);
item_kill_tree(&tree);
rcu_barrier();
}
+static long long __benchmark_split(unsigned long index,
+ int old_order, int new_order)
+{
+ struct timespec start, finish;
+ long long nsec;
+ RADIX_TREE(tree, GFP_ATOMIC);
+
+ item_insert_order(&tree, index, old_order);
+
+ clock_gettime(CLOCK_MONOTONIC, &start);
+ radix_tree_split(&tree, index, new_order);
+ clock_gettime(CLOCK_MONOTONIC, &finish);
+ nsec = (finish.tv_sec - start.tv_sec) * NSEC_PER_SEC +
+ (finish.tv_nsec - start.tv_nsec);
+
+ item_kill_tree(&tree);
+
+ return nsec;
+
+}
+
+static void benchmark_split(unsigned long size, unsigned long step)
+{
+ int i, j, idx;
+ long long nsec = 0;
+
+
+ for (idx = 0; idx < size; idx += step) {
+ for (i = 3; i < 11; i++) {
+ for (j = 0; j < i; j++) {
+ nsec += __benchmark_split(idx, i, j);
+ }
+ }
+ }
+
+ printv(2, "Size %8ld, step %8ld, split time %10lld ns\n",
+ size, step, nsec);
+
+}
+
+static long long __benchmark_join(unsigned long index,
+ unsigned order1, unsigned order2)
+{
+ unsigned long loc;
+ struct timespec start, finish;
+ long long nsec;
+ void *item, *item2 = item_create(index + 1, order1);
+ RADIX_TREE(tree, GFP_KERNEL);
+
+ item_insert_order(&tree, index, order2);
+ item = radix_tree_lookup(&tree, index);
+
+ clock_gettime(CLOCK_MONOTONIC, &start);
+ radix_tree_join(&tree, index + 1, order1, item2);
+ clock_gettime(CLOCK_MONOTONIC, &finish);
+ nsec = (finish.tv_sec - start.tv_sec) * NSEC_PER_SEC +
+ (finish.tv_nsec - start.tv_nsec);
+
+ loc = find_item(&tree, item);
+ if (loc == -1)
+ free(item);
+
+ item_kill_tree(&tree);
+
+ return nsec;
+}
+
+static void benchmark_join(unsigned long step)
+{
+ int i, j, idx;
+ long long nsec = 0;
+
+ for (idx = 0; idx < 1 << 10; idx += step) {
+ for (i = 1; i < 15; i++) {
+ for (j = 0; j < i; j++) {
+ nsec += __benchmark_join(idx, i, j);
+ }
+ }
+ }
+
+ printv(2, "Size %8d, step %8ld, join time %10lld ns\n",
+ 1 << 10, step, nsec);
+}
+
void benchmark(void)
{
unsigned long size[] = {1 << 10, 1 << 20, 0};
for (c = 0; size[c]; c++)
for (s = 0; step[s]; s++)
benchmark_size(size[c], step[s] << 9, 9);
+
+ for (c = 0; size[c]; c++)
+ for (s = 0; step[s]; s++)
+ benchmark_split(size[c], step[s]);
+
+ for (s = 0; step[s]; s++)
+ benchmark_join(step[s]);
}
idr_destroy(&idr);
}
+void idr_get_next_test(void)
+{
+ unsigned long i;
+ int nextid;
+ DEFINE_IDR(idr);
+
+ int indices[] = {4, 7, 9, 15, 65, 128, 1000, 99999, 0};
+
+ for(i = 0; indices[i]; i++) {
+ struct item *item = item_create(indices[i], 0);
+ assert(idr_alloc(&idr, item, indices[i], indices[i+1],
+ GFP_KERNEL) == indices[i]);
+ }
+
+ for(i = 0, nextid = 0; indices[i]; i++) {
+ idr_get_next(&idr, &nextid);
+ assert(nextid == indices[i]);
+ nextid++;
+ }
+
+ idr_for_each(&idr, item_idr_free, &idr);
+ idr_destroy(&idr);
+}
+
void idr_checks(void)
{
unsigned long i;
idr_alloc_test();
idr_null_test();
idr_nowait_test();
+ idr_get_next_test();
}
/*
{
DEFINE_IDA(ida);
DECLARE_BITMAP(bitmap, 2048);
- int id;
+ int id, err;
unsigned int i;
time_t s = time(NULL);
ida_remove(&ida, bit);
} else {
__set_bit(bit, bitmap);
- ida_pre_get(&ida, GFP_KERNEL);
- assert(!ida_get_new_above(&ida, bit, &id));
+ do {
+ ida_pre_get(&ida, GFP_KERNEL);
+ err = ida_get_new_above(&ida, bit, &id);
+ } while (err == -ENOMEM);
+ assert(!err);
assert(id == bit);
}
}
goto repeat;
}
+void ida_simple_get_remove_test(void)
+{
+ DEFINE_IDA(ida);
+ unsigned long i;
+
+ for (i = 0; i < 10000; i++) {
+ assert(ida_simple_get(&ida, 0, 20000, GFP_KERNEL) == i);
+ }
+ assert(ida_simple_get(&ida, 5, 30, GFP_KERNEL) < 0);
+
+ for (i = 0; i < 10000; i++) {
+ ida_simple_remove(&ida, i);
+ }
+ assert(ida_is_empty(&ida));
+
+ ida_destroy(&ida);
+}
+
void ida_checks(void)
{
DEFINE_IDA(ida);
ida_check_max();
ida_check_conv();
ida_check_random();
+ ida_simple_get_remove_test();
radix_tree_cpu_dead(1);
}
+static void *ida_random_fn(void *arg)
+{
+ rcu_register_thread();
+ ida_check_random();
+ rcu_unregister_thread();
+ return NULL;
+}
+
+void ida_thread_tests(void)
+{
+ pthread_t threads[10];
+ int i;
+
+ for (i = 0; i < ARRAY_SIZE(threads); i++)
+ if (pthread_create(&threads[i], NULL, ida_random_fn, NULL)) {
+ perror("creating ida thread");
+ exit(1);
+ }
+
+ while (i--)
+ pthread_join(threads[i], NULL);
+}
+
int __weak main(void)
{
radix_tree_init();
idr_checks();
ida_checks();
+ ida_thread_tests();
+ radix_tree_cpu_dead(1);
rcu_barrier();
if (nr_allocated)
printf("nr_allocated = %d\n", nr_allocated);
iteration_test(0, 10 + 90 * long_run);
iteration_test(7, 10 + 90 * long_run);
single_thread_tests(long_run);
+ ida_thread_tests();
/* Free any remaining preallocated nodes */
radix_tree_cpu_dead(0);
item_kill_tree(&tree);
}
+void radix_tree_clear_tags_test(void)
+{
+ unsigned long index;
+ struct radix_tree_node *node;
+ struct radix_tree_iter iter;
+ void **slot;
+
+ RADIX_TREE(tree, GFP_KERNEL);
+
+ item_insert(&tree, 0);
+ item_tag_set(&tree, 0, 0);
+ __radix_tree_lookup(&tree, 0, &node, &slot);
+ radix_tree_clear_tags(&tree, node, slot);
+ assert(item_tag_get(&tree, 0, 0) == 0);
+
+ for (index = 0; index < 1000; index++) {
+ item_insert(&tree, index);
+ item_tag_set(&tree, index, 0);
+ }
+
+ radix_tree_for_each_slot(slot, &tree, &iter, 0) {
+ radix_tree_clear_tags(&tree, iter.node, slot);
+ assert(item_tag_get(&tree, iter.index, 0) == 0);
+ }
+
+ item_kill_tree(&tree);
+}
+
void tag_check(void)
{
single_check();
thrash_tags();
rcu_barrier();
printv(2, "after thrash_tags: %d allocated\n", nr_allocated);
+ radix_tree_clear_tags_test();
}
void benchmark(void);
void idr_checks(void);
void ida_checks(void);
+void ida_thread_tests(void);
struct item *
item_tag_set(struct radix_tree_root *root, unsigned long index, int tag);
TARGETS += seccomp
TARGETS += sigaltstack
TARGETS += size
+TARGETS += splice
TARGETS += static_keys
TARGETS += sync
TARGETS += sysctl
# Makefile can operate with or without the kbuild infrastructure.
CC := $(CROSS_COMPILE)gcc
+ifeq (0,$(MAKELEVEL))
+OUTPUT := $(shell pwd)
+endif
+
TEST_GEN_PROGS := $(patsubst %,$(OUTPUT)/%,$(TEST_GEN_PROGS))
TEST_GEN_FILES := $(patsubst %,$(OUTPUT)/%,$(TEST_GEN_FILES))
rc = run_test(test_function, name);
- if (rc == MAGIC_SKIP_RETURN_VALUE)
+ if (rc == MAGIC_SKIP_RETURN_VALUE) {
test_skip(name);
- else
+ /* so that skipped test is not marked as failed */
+ rc = 0;
+ } else
test_finish(name, rc);
return rc;
--- /dev/null
+TEST_PROGS := default_file_splice_read.sh
+EXTRA := default_file_splice_read
+all: $(TEST_PROGS) $(EXTRA)
+
+include ../lib.mk
+
+clean:
+ rm -fr $(TEST_PROGS) $(EXTRA)
--- /dev/null
+#define _GNU_SOURCE
+#include <fcntl.h>
+
+int main(int argc, char **argv)
+{
+ splice(0, 0, 1, 0, 1<<30, 0);
+ return 0;
+}
--- /dev/null
+#!/bin/sh
+n=`./default_file_splice_read </dev/null | wc -c`
+
+test "$n" = 0 && exit 0
+
+echo "default_file_splice_read broken: leaked $n"
+exit 1
# Makefile for vm selftests
+ifndef OUTPUT
+ OUTPUT := $(shell pwd)
+endif
+
CFLAGS = -Wall -I ../../../../usr/include $(EXTRA_CFLAGS)
LDLIBS = -lrt
TEST_GEN_FILES = compaction_test
.PHONY: all all_32 all_64 warn_32bit_failure clean
TARGETS_C_BOTHBITS := single_step_syscall sysret_ss_attrs syscall_nt ptrace_syscall test_mremap_vdso \
- check_initial_reg_state sigreturn ldt_gdt iopl mpx-mini-test \
+ check_initial_reg_state sigreturn ldt_gdt iopl mpx-mini-test ioperm \
protection_keys test_vdso
TARGETS_C_32BIT_ONLY := entry_from_vm86 syscall_arg_fault test_syscall_vdso unwind_vdso \
test_FCMOV test_FCOMI test_FISTTP \
long ret;
asm volatile ("int $0x80"
: "=a" (ret) : "a" (243), "b" (low_desc)
- : "flags");
+ : "r8", "r9", "r10", "r11");
memcpy(&desc, low_desc, sizeof(desc));
munmap(low_desc, sizeof(desc));
--- /dev/null
+/*
+ * ioperm.c - Test case for ioperm(2)
+ * Copyright (c) 2015 Andrew Lutomirski
+ */
+
+#define _GNU_SOURCE
+#include <err.h>
+#include <stdio.h>
+#include <stdint.h>
+#include <signal.h>
+#include <setjmp.h>
+#include <stdlib.h>
+#include <string.h>
+#include <errno.h>
+#include <unistd.h>
+#include <sys/types.h>
+#include <sys/wait.h>
+#include <stdbool.h>
+#include <sched.h>
+#include <sys/io.h>
+
+static int nerrs = 0;
+
+static void sethandler(int sig, void (*handler)(int, siginfo_t *, void *),
+ int flags)
+{
+ struct sigaction sa;
+ memset(&sa, 0, sizeof(sa));
+ sa.sa_sigaction = handler;
+ sa.sa_flags = SA_SIGINFO | flags;
+ sigemptyset(&sa.sa_mask);
+ if (sigaction(sig, &sa, 0))
+ err(1, "sigaction");
+
+}
+
+static void clearhandler(int sig)
+{
+ struct sigaction sa;
+ memset(&sa, 0, sizeof(sa));
+ sa.sa_handler = SIG_DFL;
+ sigemptyset(&sa.sa_mask);
+ if (sigaction(sig, &sa, 0))
+ err(1, "sigaction");
+}
+
+static jmp_buf jmpbuf;
+
+static void sigsegv(int sig, siginfo_t *si, void *ctx_void)
+{
+ siglongjmp(jmpbuf, 1);
+}
+
+static bool try_outb(unsigned short port)
+{
+ sethandler(SIGSEGV, sigsegv, SA_RESETHAND);
+ if (sigsetjmp(jmpbuf, 1) != 0) {
+ return false;
+ } else {
+ asm volatile ("outb %%al, %w[port]"
+ : : [port] "Nd" (port), "a" (0));
+ return true;
+ }
+ clearhandler(SIGSEGV);
+}
+
+static void expect_ok(unsigned short port)
+{
+ if (!try_outb(port)) {
+ printf("[FAIL]\toutb to 0x%02hx failed\n", port);
+ exit(1);
+ }
+
+ printf("[OK]\toutb to 0x%02hx worked\n", port);
+}
+
+static void expect_gp(unsigned short port)
+{
+ if (try_outb(port)) {
+ printf("[FAIL]\toutb to 0x%02hx worked\n", port);
+ exit(1);
+ }
+
+ printf("[OK]\toutb to 0x%02hx failed\n", port);
+}
+
+int main(void)
+{
+ cpu_set_t cpuset;
+ CPU_ZERO(&cpuset);
+ CPU_SET(0, &cpuset);
+ if (sched_setaffinity(0, sizeof(cpuset), &cpuset) != 0)
+ err(1, "sched_setaffinity to CPU 0");
+
+ expect_gp(0x80);
+ expect_gp(0xed);
+
+ /*
+ * Probe for ioperm support. Note that clearing ioperm bits
+ * works even as nonroot.
+ */
+ printf("[RUN]\tenable 0x80\n");
+ if (ioperm(0x80, 1, 1) != 0) {
+ printf("[OK]\tioperm(0x80, 1, 1) failed (%d) -- try running as root\n",
+ errno);
+ return 0;
+ }
+ expect_ok(0x80);
+ expect_gp(0xed);
+
+ printf("[RUN]\tdisable 0x80\n");
+ if (ioperm(0x80, 1, 0) != 0) {
+ printf("[FAIL]\tioperm(0x80, 1, 0) failed (%d)", errno);
+ return 1;
+ }
+ expect_gp(0x80);
+ expect_gp(0xed);
+
+ /* Make sure that fork() preserves ioperm. */
+ if (ioperm(0x80, 1, 1) != 0) {
+ printf("[FAIL]\tioperm(0x80, 1, 0) failed (%d)", errno);
+ return 1;
+ }
+
+ pid_t child = fork();
+ if (child == -1)
+ err(1, "fork");
+
+ if (child == 0) {
+ printf("[RUN]\tchild: check that we inherited permissions\n");
+ expect_ok(0x80);
+ expect_gp(0xed);
+ return 0;
+ } else {
+ int status;
+ if (waitpid(child, &status, 0) != child ||
+ !WIFEXITED(status)) {
+ printf("[FAIL]\tChild died\n");
+ nerrs++;
+ } else if (WEXITSTATUS(status) != 0) {
+ printf("[FAIL]\tChild failed\n");
+ nerrs++;
+ } else {
+ printf("[OK]\tChild succeeded\n");
+ }
+ }
+
+ /* Test the capability checks. */
+
+ printf("\tDrop privileges\n");
+ if (setresuid(1, 1, 1) != 0) {
+ printf("[WARN]\tDropping privileges failed\n");
+ return 0;
+ }
+
+ printf("[RUN]\tdisable 0x80\n");
+ if (ioperm(0x80, 1, 0) != 0) {
+ printf("[FAIL]\tioperm(0x80, 1, 0) failed (%d)", errno);
+ return 1;
+ }
+ printf("[OK]\tit worked\n");
+
+ printf("[RUN]\tenable 0x80 again\n");
+ if (ioperm(0x80, 1, 1) == 0) {
+ printf("[FAIL]\tit succeeded but should have failed.\n");
+ return 1;
+ }
+ printf("[OK]\tit failed\n");
+ return 0;
+}
#define AR_DB (1 << 22)
#define AR_G (1 << 23)
+#ifdef __x86_64__
+# define INT80_CLOBBERS "r8", "r9", "r10", "r11"
+#else
+# define INT80_CLOBBERS
+#endif
+
static int nerrs;
/* Points to an array of 1024 ints, each holding its own index. */
asm volatile ("int $0x80"
: "=a" (ret), "+m" (low_user_desc) :
"a" (243), "b" (low_user_desc)
- : "flags");
+ : INT80_CLOBBERS);
return ret;
}
"+a" (eax)
: "m" (low_user_desc_clear),
[arg1] "r" ((unsigned int)(unsigned long)low_user_desc_clear)
- : "flags");
+ : INT80_CLOBBERS);
if (sel != 0) {
result = "FAIL";
"+a" (eax)
: "m" (low_user_desc_clear),
[arg1] "r" ((unsigned int)(unsigned long)low_user_desc_clear)
- : "flags");
+ : INT80_CLOBBERS);
if (sel != 0) {
result = "FAIL";
"+a" (eax)
: "m" (low_user_desc_clear),
[arg1] "r" ((unsigned int)(unsigned long)low_user_desc_clear)
- : "flags");
+ : INT80_CLOBBERS);
#ifdef __x86_64__
syscall(SYS_arch_prctl, ARCH_GET_FS, &new_base);
"+a" (eax)
: "m" (low_user_desc_clear),
[arg1] "r" ((unsigned int)(unsigned long)low_user_desc_clear)
- : "flags");
+ : INT80_CLOBBERS);
#ifdef __x86_64__
syscall(SYS_arch_prctl, ARCH_GET_GS, &new_base);
asm volatile ("int $0x80"
: "+a" (args->nr),
"+b" (args->arg0), "+c" (args->arg1), "+d" (args->arg2),
- "+S" (args->arg3), "+D" (args->arg4), "+r" (bp));
+ "+S" (args->arg3), "+D" (args->arg4), "+r" (bp)
+ : : "r8", "r9", "r10", "r11");
args->arg5 = bp;
#else
sys32_helper(args, int80_and_ret);
#ifdef __x86_64__
# define REG_IP REG_RIP
# define WIDTH "q"
+# define INT80_CLOBBERS "r8", "r9", "r10", "r11"
#else
# define REG_IP REG_EIP
# define WIDTH "l"
+# define INT80_CLOBBERS
#endif
static unsigned long get_eflags(void)
printf("[RUN]\tSet TF and check int80\n");
set_eflags(get_eflags() | X86_EFLAGS_TF);
- asm volatile ("int $0x80" : "=a" (tmp) : "a" (SYS_getpid));
+ asm volatile ("int $0x80" : "=a" (tmp) : "a" (SYS_getpid)
+ : INT80_CLOBBERS);
check_result();
/*
return ret;
}
-static unsigned long vgic_mmio_read_its_ctlr(struct kvm *vcpu,
- struct vgic_its *its,
- gpa_t addr, unsigned int len)
-{
- u32 reg = 0;
-
- mutex_lock(&its->cmd_lock);
- if (its->creadr == its->cwriter)
- reg |= GITS_CTLR_QUIESCENT;
- if (its->enabled)
- reg |= GITS_CTLR_ENABLE;
- mutex_unlock(&its->cmd_lock);
-
- return reg;
-}
-
-static void vgic_mmio_write_its_ctlr(struct kvm *kvm, struct vgic_its *its,
- gpa_t addr, unsigned int len,
- unsigned long val)
-{
- its->enabled = !!(val & GITS_CTLR_ENABLE);
-}
-
static unsigned long vgic_mmio_read_its_typer(struct kvm *kvm,
struct vgic_its *its,
gpa_t addr, unsigned int len)
#define ITS_CMD_SIZE 32
#define ITS_CMD_OFFSET(reg) ((reg) & GENMASK(19, 5))
-/*
- * By writing to CWRITER the guest announces new commands to be processed.
- * To avoid any races in the first place, we take the its_cmd lock, which
- * protects our ring buffer variables, so that there is only one user
- * per ITS handling commands at a given time.
- */
-static void vgic_mmio_write_its_cwriter(struct kvm *kvm, struct vgic_its *its,
- gpa_t addr, unsigned int len,
- unsigned long val)
+/* Must be called with the cmd_lock held. */
+static void vgic_its_process_commands(struct kvm *kvm, struct vgic_its *its)
{
gpa_t cbaser;
u64 cmd_buf[4];
- u32 reg;
- if (!its)
- return;
-
- mutex_lock(&its->cmd_lock);
-
- reg = update_64bit_reg(its->cwriter, addr & 7, len, val);
- reg = ITS_CMD_OFFSET(reg);
- if (reg >= ITS_CMD_BUFFER_SIZE(its->cbaser)) {
- mutex_unlock(&its->cmd_lock);
+ /* Commands are only processed when the ITS is enabled. */
+ if (!its->enabled)
return;
- }
- its->cwriter = reg;
cbaser = CBASER_ADDRESS(its->cbaser);
while (its->cwriter != its->creadr) {
if (its->creadr == ITS_CMD_BUFFER_SIZE(its->cbaser))
its->creadr = 0;
}
+}
+
+/*
+ * By writing to CWRITER the guest announces new commands to be processed.
+ * To avoid any races in the first place, we take the its_cmd lock, which
+ * protects our ring buffer variables, so that there is only one user
+ * per ITS handling commands at a given time.
+ */
+static void vgic_mmio_write_its_cwriter(struct kvm *kvm, struct vgic_its *its,
+ gpa_t addr, unsigned int len,
+ unsigned long val)
+{
+ u64 reg;
+
+ if (!its)
+ return;
+
+ mutex_lock(&its->cmd_lock);
+
+ reg = update_64bit_reg(its->cwriter, addr & 7, len, val);
+ reg = ITS_CMD_OFFSET(reg);
+ if (reg >= ITS_CMD_BUFFER_SIZE(its->cbaser)) {
+ mutex_unlock(&its->cmd_lock);
+ return;
+ }
+ its->cwriter = reg;
+
+ vgic_its_process_commands(kvm, its);
mutex_unlock(&its->cmd_lock);
}
*regptr = reg;
}
+static unsigned long vgic_mmio_read_its_ctlr(struct kvm *vcpu,
+ struct vgic_its *its,
+ gpa_t addr, unsigned int len)
+{
+ u32 reg = 0;
+
+ mutex_lock(&its->cmd_lock);
+ if (its->creadr == its->cwriter)
+ reg |= GITS_CTLR_QUIESCENT;
+ if (its->enabled)
+ reg |= GITS_CTLR_ENABLE;
+ mutex_unlock(&its->cmd_lock);
+
+ return reg;
+}
+
+static void vgic_mmio_write_its_ctlr(struct kvm *kvm, struct vgic_its *its,
+ gpa_t addr, unsigned int len,
+ unsigned long val)
+{
+ mutex_lock(&its->cmd_lock);
+
+ its->enabled = !!(val & GITS_CTLR_ENABLE);
+
+ /*
+ * Try to process any pending commands. This function bails out early
+ * if the ITS is disabled or no commands have been queued.
+ */
+ vgic_its_process_commands(kvm, its);
+
+ mutex_unlock(&its->cmd_lock);
+}
+
#define REGISTER_ITS_DESC(off, rd, wr, length, acc) \
{ \
.reg_offset = off, \
static void vgic_mmio_change_active(struct kvm_vcpu *vcpu, struct vgic_irq *irq,
bool new_active_state)
{
+ struct kvm_vcpu *requester_vcpu;
spin_lock(&irq->irq_lock);
+
+ /*
+ * The vcpu parameter here can mean multiple things depending on how
+ * this function is called; when handling a trap from the kernel it
+ * depends on the GIC version, and these functions are also called as
+ * part of save/restore from userspace.
+ *
+ * Therefore, we have to figure out the requester in a reliable way.
+ *
+ * When accessing VGIC state from user space, the requester_vcpu is
+ * NULL, which is fine, because we guarantee that no VCPUs are running
+ * when accessing VGIC state from user space so irq->vcpu->cpu is
+ * always -1.
+ */
+ requester_vcpu = kvm_arm_get_running_vcpu();
+
/*
* If this virtual IRQ was written into a list register, we
* have to make sure the CPU that runs the VCPU thread has
- * synced back LR state to the struct vgic_irq. We can only
- * know this for sure, when either this irq is not assigned to
- * anyone's AP list anymore, or the VCPU thread is not
- * running on any CPUs.
+ * synced back the LR state to the struct vgic_irq.
*
- * In the opposite case, we know the VCPU thread may be on its
- * way back from the guest and still has to sync back this
- * IRQ, so we release and re-acquire the spin_lock to let the
- * other thread sync back the IRQ.
+ * As long as the conditions below are true, we know the VCPU thread
+ * may be on its way back from the guest (we kicked the VCPU thread in
+ * vgic_change_active_prepare) and still has to sync back this IRQ,
+ * so we release and re-acquire the spin_lock to let the other thread
+ * sync back the IRQ.
*/
while (irq->vcpu && /* IRQ may have state in an LR somewhere */
+ irq->vcpu != requester_vcpu && /* Current thread is not the VCPU thread */
irq->vcpu->cpu != -1) /* VCPU thread is running */
cond_resched_lock(&irq->irq_lock);
/*
* If we are emulating a GICv3, we do it in an non-GICv2-compatible
* way, so we force SRE to 1 to demonstrate this to the guest.
+ * Also, we don't support any form of IRQ/FIQ bypass.
* This goes with the spec allowing the value to be RAO/WI.
*/
if (vcpu->kvm->arch.vgic.vgic_model == KVM_DEV_TYPE_ARM_VGIC_V3) {
- vgic_v3->vgic_sre = ICC_SRE_EL1_SRE;
+ vgic_v3->vgic_sre = (ICC_SRE_EL1_DIB |
+ ICC_SRE_EL1_DFB |
+ ICC_SRE_EL1_SRE);
vcpu->arch.vgic_cpu.pendbaser = INITIAL_PENDBASER_VALUE;
} else {
vgic_v3->vgic_sre = 0;
#include <linux/slab.h>
#include <linux/module.h>
#include <linux/mmu_context.h>
+#include <linux/sched/mm.h>
#include "async_pf.h"
#include <trace/events/kvm.h>
#include <linux/file.h>
#include <linux/syscore_ops.h>
#include <linux/cpu.h>
-#include <linux/sched.h>
+#include <linux/sched/signal.h>
+#include <linux/sched/mm.h>
+#include <linux/sched/stat.h>
#include <linux/cpumask.h>
#include <linux/smp.h>
#include <linux/anon_inodes.h>
mutex_init(&kvm->lock);
mutex_init(&kvm->irq_lock);
mutex_init(&kvm->slots_lock);
- atomic_set(&kvm->users_count, 1);
+ refcount_set(&kvm->users_count, 1);
INIT_LIST_HEAD(&kvm->devices);
r = kvm_arch_init_vm(kvm, type);
void kvm_get_kvm(struct kvm *kvm)
{
- atomic_inc(&kvm->users_count);
+ refcount_inc(&kvm->users_count);
}
EXPORT_SYMBOL_GPL(kvm_get_kvm);
void kvm_put_kvm(struct kvm *kvm)
{
- if (atomic_dec_and_test(&kvm->users_count))
+ if (refcount_dec_and_test(&kvm->users_count))
kvm_destroy_vm(kvm);
}
EXPORT_SYMBOL_GPL(kvm_put_kvm);
* To avoid the race between open and the removal of the debugfs
* directory we test against the users count.
*/
- if (!atomic_add_unless(&stat_data->kvm->users_count, 1, 0))
+ if (!refcount_inc_not_zero(&stat_data->kvm->users_count))
return -ENOENT;
if (simple_attr_open(inode, file, get, set, fmt)) {
projects / mirror_ubuntu-artful-kernel.git / commitdiff